mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-14 17:06:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #602 from rcordovano/parallel_file_ingest

Browse Source 
Parallel file ingest
This commit is contained in:
Richard Cordovano 2014-04-09 11:19:36 -04:00
commit 379b1ddfc3
7 changed files with 118 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java
View File

@ -73,6 +73,7 @@ import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskException;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
/**
* Top component which displays something.
*/
@ -543,8 +544,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
// The current case has been closed. Reset the ExplorerManager.
Node emptyNode = new AbstractNode(Children.LEAF);
em.setRootContext(emptyNode);
}
else if (newValue != null) {
} else if (newValue != null) {
// A new case has been opened. Reset the forward and back
// buttons. Note that a call to CoreComponentControl.openCoreWindows()
// by the new Case object will lead to a componentOpened() call
@ -598,7 +598,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
refreshTree(event.getArtifactType());
}
});
} else if (changed.equals(IngestEvent.COMPLETED.toString())) {
} else if (changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
|| changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
@ -668,8 +669,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Exception while calling Content.getUniquePath() for node: " + originNode);
}
}
else if (originNode.getLookup().lookup(String.class) != null) {
} else if (originNode.getLookup().lookup(String.class) != null) {
displayName = originNode.getLookup().lookup(String.class);
}
dataResult.setPath(displayName);
@ -752,8 +752,6 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
pcs.removePropertyChangeListener(listener);
}
/**
* Gets the tree on this DirectoryTreeTopComponent.
*
@ -881,13 +879,11 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
while (null == selectedNode && !selectedNodePath.isEmpty()) {
try {
selectedNode = NodeOp.findPath(em.getRootContext(), selectedNodePath.toArray(new String[0]));
}
catch (NodeNotFoundException ex) {
} catch (NodeNotFoundException ex) {
// The selected node may have been deleted (e.g., a deleted tag), so truncate the path and try again.
if (selectedNodePath.size() > 1) {
selectedNodePath.remove(selectedNodePath.size() - 1);
}
else {
} else {
StringBuilder nodePath = new StringBuilder();
for (int i = 0; i < previouslySelectedNodePath.length; ++i) {
nodePath.append(previouslySelectedNodePath[i]).append("/");
@ -906,8 +902,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
}
try {
em.setExploredContextAndSelection(selectedNode, new Node[]{selectedNode});
}
catch (PropertyVetoException ex) {
} catch (PropertyVetoException ex) {
logger.log(Level.WARNING, "Property veto from ExplorerManager setting selection to " + selectedNode.getName(), ex);
}
}
@ -970,8 +965,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
} catch (TskException ex) {
logger.log(Level.WARNING, "Error retrieving attributes", ex);
}
} else if ( type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT) ||
type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT) ) {
} else if (type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)
|| type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT)) {
Node interestingItemsRootNode = resultsChilds.findChild(type.getLabel());
Children interestingItemsRootChildren = interestingItemsRootNode.getChildren();
try {
@ -1033,8 +1028,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
try {
firePropertyChange(BlackboardResultViewer.FINISHED_DISPLAY_EVT, 0, 1);
}
catch (Exception e) {
} catch (Exception e) {
logger.log(Level.SEVERE, "DirectoryTreeTopComponent listener threw exception", e);
MessageNotifyUtil.Notify.show(NbBundle.getMessage(this.getClass(), "DirectoryTreeTopComponent.moduleErr"),
NbBundle.getMessage(this.getClass(),

2
Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java
View File

@ -89,7 +89,7 @@ final class FileIngestPipeline {
}
file.close();
if (!job.isCancelled()) {
IngestManager.fireFileDone(file.getId());
IngestManager.fireFileIngestDone(file.getId());
}
return errors;
}

55
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -133,7 +133,7 @@ public class IngestManager {
}
}
synchronized void cancelIngestJobs() {
void cancelIngestJobs() {
new IngestCancellationWorker().execute();
}
@ -143,29 +143,24 @@ public class IngestManager {
public enum IngestEvent {
/**
* Event sent when an ingest module has been started. Second argument of
* the property change is a string form of the module name and the third
* argument is null.
* Property change event fired when an ingest job is started. The ingest
* job id is in old value field of the PropertyChangeEvent object.
*/
STARTED,
INGEST_JOB_STARTED,
/**
* Event sent when an ingest module has completed processing by its own
* means. Second argument of the property change is a string form of the
* module name and the third argument is null.
*
* This event is generally used by listeners to perform a final data
* view refresh (listeners need to query all data from the blackboard).
* Property change event fired when an ingest job is completed. The
* ingest job id is in old value field of the PropertyChangeEvent
* object.
*/
COMPLETED,
INGEST_JOB_COMPLETED,
/**
* Event sent when an ingest module has stopped processing, and likely
* not all data has been processed. Second argument of the property
* change is a string form of the module name and third argument is
* null.
* Property change event fired when an ingest job is canceled. The
* ingest job id is in old value field of the PropertyChangeEvent
* object.
*/
STOPPED,
INGEST_JOB_CANCELLED,
/**
* Event sent when ingest module posts new data to blackboard or
* Event sent when an ingest module posts new data to blackboard or
* somewhere else. Second argument of the property change fired contains
* ModuleDataEvent object and third argument is null. The object can
* contain encapsulated new data created by the module. Listener can
@ -198,9 +193,9 @@ public class IngestManager {
pcs.removePropertyChangeListener(listener);
}
static void fireModuleEvent(String eventType, String moduleName) {
static void fireIngestJobEvent(String eventType, long jobId) {
try {
pcs.firePropertyChange(eventType, moduleName, null);
pcs.firePropertyChange(eventType, jobId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e);
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
@ -212,11 +207,11 @@ public class IngestManager {
/**
* Fire event when file is done with a pipeline run
*
* @param objId ID of file that is done
* @param fileId ID of file that is done
*/
static void fireFileDone(long objId) {
static void fireFileIngestDone(long fileId) {
try {
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), objId, null);
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e);
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
@ -340,7 +335,8 @@ public class IngestManager {
}
for (Long jobId : completedJobs) {
ingestJobs.remove(jobId);
IngestJob job = ingestJobs.remove(jobId);
fireIngestJobEvent(job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED.toString() : IngestEvent.INGEST_JOB_COMPLETED.toString(), jobId);
}
}
@ -375,7 +371,7 @@ public class IngestManager {
});
progress.start(2 * dataSources.size());
int processed = 0;
int workUnitsCompleted = 0;
for (Content dataSource : dataSources) {
if (Thread.currentThread().isInterrupted()) {
break;
@ -411,17 +407,18 @@ public class IngestManager {
// Queue the data source ingest tasks for the ingest job.
final String inputName = dataSource.getName();
progress.progress("DataSource Ingest" + " " + inputName, processed);
progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); // RJCTODO: Improve
scheduler.getDataSourceScheduler().schedule(ingestJob);
progress.progress("DataSource Ingest" + " " + inputName, ++processed);
progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted);
// Queue the file ingest tasks for the ingest job.
progress.progress("File Ingest" + " " + inputName, processed);
progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted);
scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob);
progress.progress("File Ingest" + " " + inputName, ++processed);
progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted);
if (!Thread.currentThread().isInterrupted()) {
startIngestTasks();
fireIngestJobEvent(IngestEvent.INGEST_JOB_STARTED.toString(), ingestJob.getId());
}
}
} catch (Exception ex) {

14
HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java
View File

@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.hashdatabase;
import java.awt.Color;
import java.awt.Component;
import java.awt.EventQueue;
import java.awt.Frame;
import java.awt.event.KeyEvent;
import java.beans.PropertyChangeEvent;
@ -71,9 +72,14 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
if (isFileIngestStatusChangeEvent(evt)) {
if (isIngestJobEvent(evt)) {
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
updateComponents();
}
});
}
}
});
}
@ -224,8 +230,10 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
return shortenedPath;
}
private boolean isFileIngestStatusChangeEvent(PropertyChangeEvent evt) {
return evt.getPropertyName().equals(IngestManager.IngestEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.COMPLETED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.STOPPED.toString());
private boolean isIngestJobEvent(PropertyChangeEvent evt) {
return evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_STARTED.toString())
|| evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_COMPLETED.toString())
|| evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_CANCELLED.toString());
}
@Override

57
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.awt.EventQueue;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.beans.PropertyChangeEvent;
@ -52,7 +53,6 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
private static Logger logger = Logger.getLogger(KeywordSearchEditListPanel.class.getName());
private KeywordTableModel tableModel;
private KeywordList currentKeywordList;
private boolean ingestRunning;
/**
* Creates new form KeywordSearchEditListPanel
@ -101,7 +101,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
}
});
initButtons();
setButtonStates();
addWordField.setComponentPopupMenu(rightClickMenu);
ActionListener actList = new ActionListener() {
@ -124,49 +124,28 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
pasteMenuItem.addActionListener(actList);
selectAllMenuItem.addActionListener(actList);
if (IngestManager.getInstance().isIngestRunning()) {
initIngest(0);
} else {
initIngest(1);
}
setButtonStates();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName();
Object oldValue = evt.getOldValue();
if (changed.equals(IngestEvent.COMPLETED.toString())
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
initIngest(1);
} else if (changed.equals(IngestEvent.STARTED.toString())
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
initIngest(0);
} else if (changed.equals(IngestEvent.STOPPED.toString())
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
initIngest(1);
if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString())
|| changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
|| changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
setButtonStates();
}
});
}
}
});
}
/**
* Initialize this panel depending on whether ingest is running
*
* @param running case 0: ingest running case 1: ingest not running
*/
private void initIngest(int running) {
switch (running) {
case 0:
ingestRunning = true;
break;
case 1:
ingestRunning = false;
break;
}
initButtons();
}
void initButtons() {
void setButtonStates() {
boolean ingestRunning = IngestManager.getInstance().isIngestRunning();
boolean listSet = currentKeywordList != null;
boolean isLocked = !listSet ? true : currentKeywordList.isLocked();
boolean noKeywords = !listSet ? true : currentKeywordList.getKeywords().isEmpty();
@ -441,7 +420,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
chRegex.setSelected(false);
addWordField.setText("");
initButtons();
setButtonStates();
}//GEN-LAST:event_addWordButtonActionPerformed
private void deleteWordButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteWordButtonActionPerformed
@ -449,7 +428,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
tableModel.deleteSelected(keywordTable.getSelectedRows());
KeywordSearchListsXML.getCurrent().addList(currentKeywordList);
initButtons();
setButtonStates();
}
}//GEN-LAST:event_deleteWordButtonActionPerformed
@ -549,11 +528,11 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
currentKeywordList = loader.getListsL(false).get(index);
tableModel.resync();
initButtons();
setButtonStates();
} else {
currentKeywordList = null;
tableModel.resync();
initButtons();
setButtonStates();
}
}

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java
View File

@ -44,7 +44,7 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl
if (KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.title"), NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.body"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) {
String toDelete = editListPanel.getCurrentKeywordList().getName();
editListPanel.setCurrentKeywordList(null);
editListPanel.initButtons();
editListPanel.setButtonStates();
// RJCTODO: Move this into a deleteList method in the manager
KeywordSearchListsXML deleter = KeywordSearchListsXML.getCurrent();
deleter.deleteList(toDelete);

49
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java
View File

@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearch;
import java.awt.Component;
import java.awt.Cursor;
import java.awt.EventQueue;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.beans.PropertyChangeEvent;
@ -114,26 +115,23 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
}
});
if (IngestManager.getInstance().isIngestRunning()) {
initIngest(true);
} else {
initIngest(false);
}
ingestRunning = IngestManager.getInstance().isIngestRunning();
updateComponents();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName();
Object oldValue = evt.getOldValue();
if (changed.equals(IngestEvent.COMPLETED.toString())
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
initIngest(false);
} else if (changed.equals(IngestEvent.STARTED.toString())
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
initIngest(true);
} else if (changed.equals(IngestEvent.STOPPED.toString())
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
initIngest(false);
if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString())
|| changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
|| changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
ingestRunning = IngestManager.getInstance().isIngestRunning();
updateComponents();
}
});
}
}
});
@ -152,28 +150,21 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
searchAddButton.addActionListener(searchAddListener);
}
/**
* Initialize this panel depending on whether ingest is running
*
* @param running case 0: ingest running case 1: ingest not running
*/
private void initIngest(boolean running) {
if (running) {
ingestRunning = true;
private void updateComponents() {
ingestRunning = IngestManager.getInstance().isIngestRunning();
if (ingestRunning) {
searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestTitle"));
searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestMsg" ));
listsTableModel.resync();
} else {
ingestRunning = false;
searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.searchIngestTitle"));
searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIdxSearchMsg"));
listsTableModel.resync();
}
updateIngestIndexLabel(running);
listsTableModel.resync();
updateIngestIndexLabel();
}
private void updateIngestIndexLabel(boolean ingestRunning) {
private void updateIngestIndexLabel() {
if (ingestRunning) {
ingestIndexLabel.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.ongoingIngestMsg", filesIndexed));
}
@ -184,7 +175,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
@Override
protected void postFilesIndexedChange() {
updateIngestIndexLabel(ingestRunning);
updateIngestIndexLabel();
}
/**