diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestService.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestService.java index 35be0170b4..5fcf83bda2 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestService.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestService.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.hashdatabase; +import java.beans.PropertyChangeListener; import java.io.IOException; import java.sql.SQLException; import java.util.Collections; @@ -207,6 +208,11 @@ public class HashDbIngestService implements IngestServiceFsContent { return false; } + @Override + public boolean backgroundJobsCompleteListener(PropertyChangeListener l) { + return false; + } + @Override public boolean hasSimpleConfiguration() { return false; diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java index d1b837b729..df45f8c492 100755 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest; +import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.text.DateFormat; @@ -139,7 +140,7 @@ public class IngestManager { * @param images images to execute services on */ void execute(final List services, final List images) { - logger.log(Level.INFO, "Will enqueue number of images: " + images.size()); + logger.log(Level.INFO, "Will enqueue number of images: " + images.size() + " to " + services.size() + " services."); if (!isIngestRunning()) { ui.clearMessages(); @@ -443,8 +444,9 @@ public class IngestManager { */ public static List enumerateImageServices() { List ret = new ArrayList(); - for (IngestServiceImage list : Lookup.getDefault().lookupAll(IngestServiceImage.class)) + for (IngestServiceImage list : Lookup.getDefault().lookupAll(IngestServiceImage.class)) { ret.add(list); + } return ret; } @@ -453,8 +455,9 @@ public class IngestManager { */ public static List enumerateFsContentServices() { List ret = new ArrayList(); - for (IngestServiceFsContent list : Lookup.getDefault().lookupAll(IngestServiceFsContent.class)) + for (IngestServiceFsContent list : Lookup.getDefault().lookupAll(IngestServiceFsContent.class)) { ret.add(list); + } return ret; } @@ -750,7 +753,7 @@ public class IngestManager { public synchronized String toString() { return "FsContentQueue, size: " + Integer.toString(fsContentUnits.size()); } - + public String printQueue() { StringBuilder sb = new StringBuilder(); for (QueueUnit u : fsContentUnits) { @@ -903,7 +906,7 @@ public class IngestManager { hash = 37 * hash + (this.services != null ? this.services.hashCode() : 0); return hash; } - + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -1134,18 +1137,63 @@ public class IngestManager { handleInterruption(); logger.log(Level.SEVERE, "Fatal error during ingest.", ex); } finally { - stats.end(); + //stats.end(); progress.finish(); if (!this.isCancelled()) { - logger.log(Level.INFO, "Summary Report: " + stats.toString()); - ui.displayReport(stats.toHtmlString()); + //logger.log(Level.INFO, "Summary Report: " + stats.toString()); + //ui.displayReport(stats.toHtmlString()); + new FsServicesComplete(stats); } initMainProgress(0); } } + /** + * Ensures that all background threads are done + * then finalize the stats and show dialog + */ + private class FsServicesComplete { + + private IngestManagerStats stats; //ongoing stats + private List running = new ArrayList(); + + FsServicesComplete(IngestManagerStats stats) { + this.stats = stats; + + for (IngestServiceAbstract s : fsContentServices) { + if (s.backgroundJobsCompleteListener(new PropertyChangeListener() { + + @Override + public void propertyChange(PropertyChangeEvent evt) { + if (evt.getPropertyName().equals(IngestServiceAbstract.BCKGRND_JOBS_COMPLETED_EVT)) { + IngestServiceAbstract service = (IngestServiceAbstract) evt.getNewValue(); + running.remove(service); + if (running.isEmpty()) { + showStats(); + } + } + } + })) { + running.add(s); + } + } + + //no listeners registered since no services running any longer + if (running.isEmpty()) { + showStats(); + } + + } + + void showStats() { + stats.end(); + logger.log(Level.INFO, "Summary Report: " + stats.toString()); + ui.displayReport(stats.toHtmlString()); + } + } + private void handleInterruption() { for (IngestServiceFsContent s : fsContentServices) { s.stop(); @@ -1254,9 +1302,9 @@ public class IngestManager { progress.progress(serviceName + " " + imageName, ++processed); } } - + //logger.log(Level.INFO, fsContentQueue.printQueue()); - + progress.progress("Sorting files", processed); sortFsContents(); } diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestServiceAbstract.java b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestServiceAbstract.java index fb67fec0ef..1e819cf66c 100644 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestServiceAbstract.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestServiceAbstract.java @@ -19,12 +19,15 @@ package org.sleuthkit.autopsy.ingest; +import java.beans.PropertyChangeListener; + /** * Base interface for ingest services */ public interface IngestServiceAbstract { public enum ServiceType {Image, FsContent}; + public static final String BCKGRND_JOBS_COMPLETED_EVT = "BCKGRND_JOBS_COMPLETED_EVT"; /** * notification from manager that brand new processing should be initiated. @@ -66,6 +69,16 @@ public interface IngestServiceAbstract { */ public boolean hasBackgroundJobsRunning(); + /** + * Register listener to notify when all background jobs have completed and the service + * has truly finished. The service should first check if it has threads running, and then register the listener, all in atomic operation. + * The event fired off should be BCKGRND_JOBS_COMPLETED_EVT, with the instance of IngestServiceAbstract in the newValue parameter. + * + * @param l listener + * @return true if listener registered, false otherwise (i.e. no background jobs were running) + */ + public boolean backgroundJobsCompleteListener(PropertyChangeListener l); + /** * @return does this service have a simple configuration? diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleFsContentIngestService.java b/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleFsContentIngestService.java index 41bc0b9f67..8819b72357 100644 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleFsContentIngestService.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleFsContentIngestService.java @@ -18,9 +18,9 @@ */ package org.sleuthkit.autopsy.ingest.example; +import java.beans.PropertyChangeListener; import java.util.logging.Level; import java.util.logging.Logger; -import javax.swing.JPanel; import org.sleuthkit.autopsy.ingest.IngestManagerProxy; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; @@ -117,6 +117,11 @@ public class ExampleFsContentIngestService implements IngestServiceFsContent { return false; } + @Override + public boolean backgroundJobsCompleteListener(PropertyChangeListener l) { + return false; + } + @Override public void saveAdvancedConfiguration() { } diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleImageIngestService.java b/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleImageIngestService.java index ae38dbf2ee..73ab812034 100644 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleImageIngestService.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/example/ExampleImageIngestService.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest.example; +import java.beans.PropertyChangeListener; import java.util.logging.Level; import java.util.logging.Logger; import org.sleuthkit.autopsy.ingest.IngestImageWorkerController; @@ -147,6 +148,11 @@ public final class ExampleImageIngestService implements IngestServiceImage { return false; } + @Override + public boolean backgroundJobsCompleteListener(PropertyChangeListener l) { + return false; + } + @Override public void saveAdvancedConfiguration() { } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index 8a7c5ef042..48c5f07340 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -48,6 +48,7 @@ class Ingester { } @Override + @SuppressWarnings("FinalizeDeclaration") protected void finalize() throws Throwable { super.finalize(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java index e9eada1663..1a586823bb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java @@ -18,6 +18,8 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.beans.PropertyChangeListener; +import java.beans.PropertyChangeSupport; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -25,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; -import javax.swing.JPanel; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.apache.commons.lang.StringEscapeUtils; @@ -38,6 +39,7 @@ import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManagerProxy; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; +import org.sleuthkit.autopsy.ingest.IngestServiceAbstract; import org.sleuthkit.autopsy.ingest.IngestServiceFsContent; import org.sleuthkit.autopsy.ingest.ServiceDataEvent; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; @@ -68,11 +70,15 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent private Indexer indexer; private SwingWorker searcher; private volatile boolean searcherDone = true; + private static PropertyChangeSupport pcs = null; private Map> currentResults; private volatile int messageID = 0; + private boolean processedFiles; private volatile boolean finalRun = false; + private volatile boolean finalRunComplete = false; private final String hashDBServiceName = "Hash Lookup"; private SleuthkitCase caseHandle = null; + // TODO: use a more robust method than checking file extension to determine // whether to try a file // supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika @@ -108,6 +114,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent //notify depending service that keyword search (would) encountered error for this file return ProcessResult.ERROR; } + + if (processedFiles == false) + processedFiles = true; //check if time to commit and previous search is not running //commiting while searching causes performance issues @@ -151,11 +160,12 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent updateKeywords(); //run one last search as there are probably some new files committed - if (keywords != null && !keywords.isEmpty()) { + if (keywords != null && !keywords.isEmpty() && processedFiles == true) { finalRun = true; searcher = new Searcher(keywords); searcher.execute(); } else { + finalRunComplete = true; managerProxy.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, this, "Completed")); } //postSummary(); @@ -191,6 +201,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent caseHandle = Case.getCurrentCase().getSleuthkitCase(); this.managerProxy = managerProxy; + + //this deregisters previously registered listeners at every init() + pcs = new PropertyChangeSupport(KeywordSearchIngestService.class); final Server.Core solrCore = KeywordSearch.getServer().getCore(); ingester = solrCore.getIngester(); @@ -209,7 +222,9 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent managerProxy.postMessage(IngestMessage.createWarningMessage(++messageID, instance, "No keywords in keyword list.", "Only indexing will be done and and keyword search will be skipped (it can be executed later again as ingest or using toolbar search feature).")); } + processedFiles = false; finalRun = false; + finalRunComplete = false; searcherDone = true; //make sure to start the initial searcher //keeps track of all results per run not to repeat reporting the same hits currentResults = new HashMap>(); @@ -271,6 +286,17 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent //no need to check timer thread } + + @Override + public synchronized boolean backgroundJobsCompleteListener(PropertyChangeListener l) { + if (finalRunComplete == true) + return false; + else { + pcs.addPropertyChangeListener(l); + return true; + } + + } private void commit() { ingester.commit(); @@ -468,11 +494,12 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent if (fsContent.getSize() < MAX_STRING_EXTRACT_SIZE) { if (!extractAndIngest(fsContent)) { logger.log(Level.INFO, "Failed to extract strings and ingest, file '" + fsContent.getName() + "' (id: " + fsContent.getId() + ")."); + ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED); } else { ingestStatus.put(fsContent.getId(), IngestStatus.EXTRACTED_INGESTED); } } else { - ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED); + //ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED); } } } @@ -499,6 +526,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent @Override public boolean cancel() { + finalRunComplete = true; return Searcher.this.cancel(true); } }); @@ -655,9 +683,11 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent //logger.log(Level.INFO, "Finished search"); if (finalRun) { + finalRunComplete = true; keywords.clear(); keywordLists.clear(); managerProxy.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchIngestService.instance, "Completed")); + pcs.firePropertyChange(IngestServiceAbstract.BCKGRND_JOBS_COMPLETED_EVT, null, KeywordSearchIngestService.this); } } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestService.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestService.java index 9e9332bf8e..da068b63a6 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestService.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestService.java @@ -18,12 +18,12 @@ */ package org.sleuthkit.autopsy.recentactivity; +import java.beans.PropertyChangeListener; import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import javax.swing.JPanel; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.ingest.IngestImageWorkerController; import org.sleuthkit.autopsy.ingest.IngestManager; @@ -162,4 +162,10 @@ public final class RAImageIngestService implements IngestServiceImage { public boolean hasBackgroundJobsRunning() { return false; } + + + @Override + public boolean backgroundJobsCompleteListener(PropertyChangeListener l) { + return false; + } }