Added FILE_DONE event to Ingest Manager. Comments cleanup

This commit is contained in:
Brian Carrier 2013-10-25 15:12:34 -04:00
parent 0b8420e47c
commit efbb7d0acf
2 changed files with 57 additions and 18 deletions

View File

@ -86,40 +86,52 @@ public class IngestManager {
public enum IngestModuleEvent { public enum IngestModuleEvent {
/** /**
* Event sent when the ingest module has been started processing. Second * Event sent when an ingest module has been started. Second
* argument of the property change fired contains module name String and * argument of the property change is a string form of the module name
* third argument is null. * and the third argument is null.
*/ */
STARTED, STARTED,
/** /**
* Event sent when the ingest module has completed processing. Second * Event sent when an ingest module has completed processing by its own
* argument of the property change fired contains module name String and * means. Second
* third argument is null. * argument of the property change is a string form of the module name
* and the third argument is null.
* *
* This event is generally used by listeners to perform a final data * This event is generally used by listeners to perform a final data
* view refresh (listeners need to query all data from the blackboard). * view refresh (listeners need to query all data from the blackboard).
*
*/ */
COMPLETED, COMPLETED,
/** /**
* Event sent when the ingest module has stopped processing, and likely * Event sent when an ingest module has stopped processing, and likely
* not all data has been processed. Second argument of the property * not all data has been processed. Second argument of the property
* change fired contains module name String and third argument is null. * change is a string form of the module name and third argument is null.
*/ */
STOPPED, STOPPED,
/** /**
* Event sent when ingest module has new data. Second argument of the * Event sent when ingest module posts new data to blackboard or somewhere
* else. Second argument of the
* property change fired contains ModuleDataEvent object and third * property change fired contains ModuleDataEvent object and third
* argument is null. The object can contain encapsulated new data * argument is null. The object can contain encapsulated new data
* created by the module. Listener can also query new data as needed. * created by the module. Listener can also query new data as needed.
*
*/ */
DATA, DATA,
/** /**
* Event send when content changed, either its attributes changed, or * Event send when content changed, either its attributes changed, or
* new content children have been added * new content children have been added. I.e. from ZIP files or Carved files
*/ */
CONTENT_CHANGED CONTENT_CHANGED,
/**
* Event sent when a file has finished going through a pipeline of modules.
* Second argument is the object ID. Third argument is null
*/
FILE_DONE,
}; };
//ui //ui
//Initialized by Installer in AWT thread once the Window System is ready //Initialized by Installer in AWT thread once the Window System is ready
@ -196,11 +208,29 @@ public class IngestManager {
static synchronized void fireModuleEvent(String eventType, String moduleName) { static synchronized void fireModuleEvent(String eventType, String moduleName) {
pcs.firePropertyChange(eventType, moduleName, null); pcs.firePropertyChange(eventType, moduleName, null);
} }
/**
* Fire event when file is done with a pipeline run
* @param objId ID of file that is done
*/
static synchronized void fireFileDone(long objId) {
pcs.firePropertyChange(IngestModuleEvent.FILE_DONE.toString(), objId, null);
}
/**
* Fire event for ModuleDataEvent (when modules post data to blackboard, etc.)
* @param moduleDataEvent
*/
static synchronized void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { static synchronized void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
pcs.firePropertyChange(IngestModuleEvent.DATA.toString(), moduleDataEvent, null); pcs.firePropertyChange(IngestModuleEvent.DATA.toString(), moduleDataEvent, null);
} }
/**
* Fire event for ModuleContentChanged (when modules create new content that needs to be analyzed)
* @param moduleContentEvent
*/
static synchronized void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { static synchronized void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
pcs.firePropertyChange(IngestModuleEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null); pcs.firePropertyChange(IngestModuleEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null);
} }
@ -282,7 +312,8 @@ public class IngestManager {
} }
/** /**
* Starts the needed worker threads. * Starts the File-level Ingest Module pipeline and the Data Source-level Ingest Modules
* for the queued up data sources and files.
* *
* if AbstractFile module is still running, do nothing and allow it to * if AbstractFile module is still running, do nothing and allow it to
* consume queue otherwise start /restart AbstractFile worker * consume queue otherwise start /restart AbstractFile worker
@ -303,8 +334,10 @@ public class IngestManager {
ingestMonitor.start(); ingestMonitor.start();
} }
/////////
// Start the data source-level ingest modules
List<IngestDataSourceThread> newThreads = new ArrayList<>(); List<IngestDataSourceThread> newThreads = new ArrayList<>();
//image ingesters
// cycle through each data source content in the queue // cycle through each data source content in the queue
while (dataSourceScheduler.hasNext()) { while (dataSourceScheduler.hasNext()) {
if (allInited == false) { if (allInited == false) {
@ -987,10 +1020,14 @@ public class IngestManager {
logger.log(Level.SEVERE, "Error: out of memory from module: " + module.getName(), e); logger.log(Level.SEVERE, "Error: out of memory from module: " + module.getName(), e);
stats.addError(module); stats.addError(module);
} }
} //end for every module } //end for every module
//free the internal file resource after done with every module //free the internal file resource after done with every module
fileToProcess.close(); fileToProcess.close();
// notify listeners thsi file is done
fireFileDone(fileToProcess.getId());
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
if (newTotalEnqueuedFiles > totalEnqueuedFiles) { if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
@ -1076,7 +1113,7 @@ public class IngestManager {
} }
} }
/* Thread that adds content/file and module pairs to queues */ /* Thread that adds content/file and module pairs to queues. Starts pipelines when done. */
private class EnqueueWorker extends SwingWorker<Object, Void> { private class EnqueueWorker extends SwingWorker<Object, Void> {
private List<IngestModuleAbstract> modules; private List<IngestModuleAbstract> modules;

View File

@ -110,7 +110,8 @@ public class IngestServices {
/** /**
* Fire module data event to notify registered module data event listeners * Fire module data event to notify registered module data event listeners that there
* is new data of a given type from a module
* @param moduleDataEvent module data event, encapsulating blackboard artifact data * @param moduleDataEvent module data event, encapsulating blackboard artifact data
*/ */
public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
@ -120,6 +121,7 @@ public class IngestServices {
/** /**
* Fire module content event to notify registered module content event listeners * Fire module content event to notify registered module content event listeners
* that there is new content (from ZIP file contents, carving, etc.)
* @param moduleContentEvent module content event, encapsulating content changed * @param moduleContentEvent module content event, encapsulating content changed
*/ */
public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {