Merge pull request #700 from rcordovano/ingest_improvements

Ingest improvements
This commit is contained in:
Richard Cordovano 2014-05-02 15:19:55 -04:00
commit 15a05ddb90
29 changed files with 1282 additions and 1248 deletions

View File

@ -19,7 +19,7 @@
package org.sleuthkit.autopsy.casemodule; package org.sleuthkit.autopsy.casemodule;
import org.sleuthkit.autopsy.ingest.IngestJobLauncher; import org.sleuthkit.autopsy.ingest.IngestJobConfigurator;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import java.awt.Color; import java.awt.Color;
import java.awt.Component; import java.awt.Component;
@ -46,7 +46,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor;
class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel<WizardDescriptor> { class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel<WizardDescriptor> {
private static final Logger logger = Logger.getLogger(AddImageWizardIngestConfigPanel.class.getName()); private static final Logger logger = Logger.getLogger(AddImageWizardIngestConfigPanel.class.getName());
private IngestJobLauncher ingestConfig; private IngestJobConfigurator ingestConfig;
/** /**
* The visual component that displays this panel. If you need to access the * The visual component that displays this panel. If you need to access the
* component from this class, just use getComponent(). * component from this class, just use getComponent().
@ -73,7 +73,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel<WizardDe
this.progressPanel = proPanel; this.progressPanel = proPanel;
this.dataSourcePanel = dsPanel; this.dataSourcePanel = dsPanel;
ingestConfig = new IngestJobLauncher(AddImageWizardIngestConfigPanel.class.getCanonicalName()); ingestConfig = new IngestJobConfigurator(AddImageWizardIngestConfigPanel.class.getCanonicalName());
List<String> messages = ingestConfig.getIngestJobConfigWarnings(); List<String> messages = ingestConfig.getIngestJobConfigWarnings();
if (messages.isEmpty() == false) { if (messages.isEmpty() == false) {
StringBuilder warning = new StringBuilder(); StringBuilder warning = new StringBuilder();

View File

@ -188,15 +188,12 @@
<Component class="javax.swing.JComboBox" name="numberOfFileIngestThreadsComboBox"> <Component class="javax.swing.JComboBox" name="numberOfFileIngestThreadsComboBox">
<Properties> <Properties>
<Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor"> <Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor">
<StringArray count="4"> <StringArray count="0"/>
<StringItem index="0" value="1"/>
<StringItem index="1" value="2"/>
<StringItem index="2" value="3"/>
<StringItem index="3" value="4"/>
</StringArray>
</Property> </Property>
<Property name="selectedIndex" type="int" value="1"/>
</Properties> </Properties>
<AuxValues>
<AuxValue name="JavaCodeGenerator_TypeParameters" type="java.lang.String" value="&lt;Integer&gt;"/>
</AuxValues>
</Component> </Component>
</SubComponents> </SubComponents>
</Form> </Form>

View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.corecomponents; package org.sleuthkit.autopsy.corecomponents;
import java.util.prefs.Preferences; import java.util.prefs.Preferences;
import javax.swing.DefaultComboBoxModel;
import org.openide.util.NbPreferences; import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManager;
@ -33,10 +34,35 @@ final class GeneralPanel extends javax.swing.JPanel {
GeneralPanel(GeneralOptionsPanelController controller) { GeneralPanel(GeneralOptionsPanelController controller) {
initComponents(); initComponents();
numberOfFileIngestThreadsComboBox.setModel(new DefaultComboBoxModel<>(new Integer[]{1, 2, 4, 8, 16}));
ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected());
// TODO listen to changes in form fields and call controller.changed() // TODO listen to changes in form fields and call controller.changed()
} }
void load() {
boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false);
keepCurrentViewerRB.setSelected(keepPreferredViewer);
useBestViewerRB.setSelected(!keepPreferredViewer);
boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true);
useLocalTimeRB.setSelected(useLocalTime);
useGMTTimeRB.setSelected(!useLocalTime);
dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false));
viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true));
numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getNumberOfFileIngestThreads());
}
void store() {
prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected());
prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected());
prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected());
prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected());
IngestManager.setNumberOfFileIngestThreads((Integer) numberOfFileIngestThreadsComboBox.getSelectedItem());
}
boolean valid() {
return true;
}
/** /**
* This method is called from within the constructor to initialize the form. * This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always * WARNING: Do NOT modify this code. The content of this method is always
@ -57,7 +83,7 @@ final class GeneralPanel extends javax.swing.JPanel {
dataSourcesHideKnownCB = new javax.swing.JCheckBox(); dataSourcesHideKnownCB = new javax.swing.JCheckBox();
viewsHideKnownCB = new javax.swing.JCheckBox(); viewsHideKnownCB = new javax.swing.JCheckBox();
jLabel4 = new javax.swing.JLabel(); jLabel4 = new javax.swing.JLabel();
numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox(); numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox<Integer>();
buttonGroup1.add(useBestViewerRB); buttonGroup1.add(useBestViewerRB);
useBestViewerRB.setSelected(true); useBestViewerRB.setSelected(true);
@ -97,9 +123,6 @@ final class GeneralPanel extends javax.swing.JPanel {
org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N
numberOfFileIngestThreadsComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "1", "2", "3", "4" }));
numberOfFileIngestThreadsComboBox.setSelectedIndex(1);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout); this.setLayout(layout);
layout.setHorizontalGroup( layout.setHorizontalGroup(
@ -163,33 +186,8 @@ final class GeneralPanel extends javax.swing.JPanel {
}//GEN-LAST:event_useBestViewerRBActionPerformed }//GEN-LAST:event_useBestViewerRBActionPerformed
private void useGMTTimeRBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRBActionPerformed private void useGMTTimeRBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRBActionPerformed
ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected());
}//GEN-LAST:event_useGMTTimeRBActionPerformed }//GEN-LAST:event_useGMTTimeRBActionPerformed
void load() {
boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false);
keepCurrentViewerRB.setSelected(keepPreferredViewer);
useBestViewerRB.setSelected(!keepPreferredViewer);
boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true);
useLocalTimeRB.setSelected(useLocalTime);
useGMTTimeRB.setSelected(!useLocalTime);
dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false));
viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true));
numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getInstance().getNumberOfFileIngestThreads());
}
void store() {
prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected());
prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected());
prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected());
prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected());
IngestManager.getInstance().setNumberOfFileIngestThreads(Integer.valueOf(numberOfFileIngestThreadsComboBox.getSelectedItem().toString()));
}
boolean valid() {
// TODO check whether form is consistent and complete
return true;
}
// Variables declaration - do not modify//GEN-BEGIN:variables // Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.ButtonGroup buttonGroup1; private javax.swing.ButtonGroup buttonGroup1;
private javax.swing.ButtonGroup buttonGroup3; private javax.swing.ButtonGroup buttonGroup3;
@ -199,7 +197,7 @@ final class GeneralPanel extends javax.swing.JPanel {
private javax.swing.JLabel jLabel3; private javax.swing.JLabel jLabel3;
private javax.swing.JLabel jLabel4; private javax.swing.JLabel jLabel4;
private javax.swing.JRadioButton keepCurrentViewerRB; private javax.swing.JRadioButton keepCurrentViewerRB;
private javax.swing.JComboBox numberOfFileIngestThreadsComboBox; private javax.swing.JComboBox<Integer> numberOfFileIngestThreadsComboBox;
private javax.swing.JRadioButton useBestViewerRB; private javax.swing.JRadioButton useBestViewerRB;
private javax.swing.JRadioButton useGMTTimeRB; private javax.swing.JRadioButton useGMTTimeRB;
private javax.swing.JRadioButton useLocalTimeRB; private javax.swing.JRadioButton useLocalTimeRB;

View File

@ -131,7 +131,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
private void setListener() { private void setListener() {
Case.addPropertyChangeListener(this);// add this class to listen to any changes in the Case.java class Case.addPropertyChangeListener(this);// add this class to listen to any changes in the Case.java class
this.em.addPropertyChangeListener(this); this.em.addPropertyChangeListener(this);
IngestManager.addPropertyChangeListener(this); IngestManager.getInstance().addIngestJobEventListener(this);
IngestManager.getInstance().addIngestModuleEventListener(this);
} }
public void setDirectoryListingActive() { public void setDirectoryListingActive() {

View File

@ -52,12 +52,6 @@ Ensure the Case drive has at least 1GB free space and restart ingest.
IngestJobConfigurationPanel.advancedButton.text=Advanced IngestJobConfigurationPanel.advancedButton.text=Advanced
IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\: IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\:
IngestScheduler.FileSched.toString.curFiles.text=\
CurFiles, size\:
IngestScheduler.FileSched.toString.curDirs.text=\
CurDirs(stack), size\:
IngestScheduler.FileSched.toString.rootDirs.text=\
RootDirs(sorted), size\:
IngestManager.StartIngestJobsTask.run.displayName=Queueing ingest tasks IngestManager.StartIngestJobsTask.run.displayName=Queueing ingest tasks
IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...) IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...)
IngestManager.StartIngestJobsTask.run.catchException.msg=An error occurred while starting ingest. Results may only be partial IngestManager.StartIngestJobsTask.run.catchException.msg=An error occurred while starting ingest. Results may only be partial
@ -66,13 +60,9 @@ IngestMessagePanel.sortByComboBox.model.priority=Priority
IngestMessagesToolbar.customizeButton.toolTipText=Ingest Messages IngestMessagesToolbar.customizeButton.toolTipText=Ingest Messages
IngestMessageTopComponent.initComponents.name=Ingest Inbox IngestMessageTopComponent.initComponents.name=Ingest Inbox
IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=Unable to start up one or more ingest modules, ingest job cancelled. IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=Unable to start up one or more ingest modules, ingest job cancelled.
IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=Please disable the failed modules or fix the errors and then restart ingest\ IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=Please disable the failed modules or fix the errors and then restart ingest \
by right clicking on the data source and selecting Run Ingest Modules. by right clicking on the data source and selecting Run Ingest Modules.
IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\:\ IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\: \
\ \
{0} {0}
IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=Ingest Failure IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=Ingest Failure
IngestManager.StartIngestJobsTask.run.progress.msg1=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg2=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg3=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg4=Data source ingest tasks for {0}

View File

@ -1,74 +1,64 @@
CTL_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8 CTL_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8
HINT_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8\u30A6\u30A3\u30F3\u30C9\u30A6 HINT_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8\u30a6\u30a3\u30f3\u30c9\u30a6
IngestDialog.closeButton.title=\u9589\u3058\u308B IngestDialog.closeButton.title=\u9589\u3058\u308b
IngestDialog.startButton.title=\u30B9\u30BF\u30FC\u30C8 IngestDialog.startButton.title=\u30b9\u30bf\u30fc\u30c8
IngestDialog.title.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB IngestDialog.title.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb
IngestJob.progress.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 IngestJob.progress.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09
IngestJob.progress.dataSourceIngest.displayName={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 IngestJob.progress.dataSourceIngest.displayName={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
IngestJob.progress.fileIngest.displayName={0}\u306E\u30D5\u30A1\u30A4\u30EB\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 IngestJob.progress.fileIngest.displayName={0}\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
IngestJobConfigurationPanel.advancedButton.actionCommand=\u30A2\u30C9\u30D0\u30F3\u30B9 IngestJobConfigurationPanel.advancedButton.actionCommand=\u30a2\u30c9\u30d0\u30f3\u30b9
IngestJobConfigurationPanel.advancedButton.text=\u30A2\u30C9\u30D0\u30F3\u30B9 IngestJobConfigurationPanel.advancedButton.text=\u30a2\u30c9\u30d0\u30f3\u30b9
IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u306E\u51E6\u7406 IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u306e\u51e6\u7406
IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524A\u9664\u3055\u308C\u305F\u30D5\u30A1\u30A4\u30EB\u7B49\u306E\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u51E6\u7406\u3002\u3088\u308A\u5B8C\u5168\u306A\u7D50\u679C\u304C\u51FA\u307E\u3059\u304C\u3001\u5927\u304D\u3044\u30A4\u30E1\u30FC\u30B8\u3067\u306F\u51E6\u7406\u6642\u9593\u304C\u9577\u304F\u306A\u308B\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524a\u9664\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb\u7b49\u306e\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u3092\u51e6\u7406\u3002\u3088\u308a\u5b8c\u5168\u306a\u7d50\u679c\u304c\u51fa\u307e\u3059\u304c\u3001\u5927\u304d\u3044\u30a4\u30e1\u30fc\u30b8\u3067\u306f\u51e6\u7406\u6642\u9593\u304c\u9577\u304f\u306a\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestManager.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC IngestManager.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc
IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestManager.StartIngestJobsTask.run.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 IngestManager.StartIngestJobsTask.run.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09
IngestManager.StartIngestJobsTask.run.catchException.msg=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u306E\u958B\u59CB\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002\u7D50\u679C\u304C\u4E00\u90E8\u306E\u3082\u306E IngestManager.StartIngestJobsTask.run.catchException.msg=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u306e\u958b\u59cb\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002\u7d50\u679c\u304c\u4e00\u90e8\u306e\u3082\u306e
IngestManager.StartIngestJobsTask.run.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF\u3092\u30AD\u30E5\u30FC\u30A4\u30F3\u30B0 IngestManager.StartIngestJobsTask.run.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af\u3092\u30ad\u30e5\u30fc\u30a4\u30f3\u30b0
IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30BD\u30FC\u30B9\u3001\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u3001\u8A73\u7D30\u304A\u3088\u3073\u30C7\u30FC\u30BF\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30bd\u30fc\u30b9\u3001\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u3001\u8a73\u7d30\u304a\u3088\u3073\u30c7\u30fc\u30bf\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.exception.srcSubjNotNull.msg=\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.srcSubjNotNull.msg=\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.exception.typeSrcSubjNotNull.msg=\u30E1\u30C3\u30BB\u30FC\u30B8\u30BF\u30A4\u30D7\u3001\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.typeSrcSubjNotNull.msg=\u30e1\u30c3\u30bb\u30fc\u30b8\u30bf\u30a4\u30d7\u3001\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.toString.data.text=\ \u30C7\u30FC\u30BF\uFF1A{0} IngestMessage.toString.data.text=\ \u30c7\u30fc\u30bf\uff1a{0}
IngestMessage.toString.date.text=\ \u65E5\u4ED8\uFF1A{0} IngestMessage.toString.date.text=\ \u65e5\u4ed8\uff1a{0}
IngestMessage.toString.details.text=\ \u8A73\u7D30\uFF1A{0} IngestMessage.toString.details.text=\ \u8a73\u7d30\uff1a{0}
IngestMessage.toString.subject.text=\ \u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\uFF1A{0} IngestMessage.toString.subject.text=\ \u30b5\u30d6\u30b8\u30a7\u30af\u30c8\uff1a{0}
IngestMessage.toString.type.text=\u30BF\u30A4\u30D7\uFF1A{0} IngestMessage.toString.type.text=\u30bf\u30a4\u30d7\uff1a{0}
IngestMessageDetailsPanel.copyMenuItem.text=\u30B3\u30D4\u30FC IngestMessageDetailsPanel.copyMenuItem.text=\u30b3\u30d4\u30fc
IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30C6\u30AD\u30B9\u30C8\uFF0Fhtml IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30c6\u30ad\u30b9\u30c8\uff0fhtml
IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629E IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629e
IngestMessageDetailsPanel.viewArtifactButton.text=\u7D50\u679C\u3078\u79FB\u52D5 IngestMessageDetailsPanel.viewArtifactButton.text=\u7d50\u679c\u3078\u79fb\u52d5
IngestMessageDetailsPanel.viewContentButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u3078\u79FB\u52D5 IngestMessageDetailsPanel.viewContentButton.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3078\u79fb\u52d5
IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30D6\u30FC\u30EB\u5024\u3067\u306F\u306A\u3044\u3082\u306E\u306BBooleanRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30d6\u30fc\u30eb\u5024\u3067\u306f\u306a\u3044\u3082\u306e\u306bBooleanRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f
IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65E5\u4ED8\u3067\u306F\u306A\u3044\u3082\u306E\u306BDateRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F\u3002 IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65e5\u4ed8\u3067\u306f\u306a\u3044\u3082\u306e\u306bDateRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f\u3002
IngestMessagePanel.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC IngestMessagePanel.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc
IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestMessagePanel.MsgTableMod.colNames.module=\u30E2\u30B8\u30E5\u30FC\u30EB IngestMessagePanel.MsgTableMod.colNames.module=\u30e2\u30b8\u30e5\u30fc\u30eb
IngestMessagePanel.MsgTableMod.colNames.new=\u65B0\u898F\uFF1F IngestMessagePanel.MsgTableMod.colNames.new=\u65b0\u898f\uff1f
IngestMessagePanel.MsgTableMod.colNames.num=\u756A\u53F7 IngestMessagePanel.MsgTableMod.colNames.num=\u756a\u53f7
IngestMessagePanel.MsgTableMod.colNames.subject=\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8 IngestMessagePanel.MsgTableMod.colNames.subject=\u30b5\u30d6\u30b8\u30a7\u30af\u30c8
IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30BF\u30A4\u30E0\u30B9\u30BF\u30F3\u30D7 IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30bf\u30a4\u30e0\u30b9\u30bf\u30f3\u30d7
IngestMessagePanel.sortByComboBox.model.priority=\u512A\u5148\u5EA6 IngestMessagePanel.sortByComboBox.model.priority=\u512a\u5148\u5ea6
IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593 IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593
IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uFF08\u6642\u7CFB\u5217\uFF09\u307E\u305F\u306F\u30E1\u30C3\u30BB\u30FC\u30B8\u306E\u512A\u5148\u5EA6\u3067\u30BD\u30FC\u30C8 IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uff08\u6642\u7cfb\u5217\uff09\u307e\u305f\u306f\u30e1\u30c3\u30bb\u30fc\u30b8\u306e\u512a\u5148\u5ea6\u3067\u30bd\u30fc\u30c8
IngestMessagePanel.sortByLabel.text=\u4E0B\u8A18\u3067\u30BD\u30FC\u30C8\uFF1A IngestMessagePanel.sortByLabel.text=\u4e0b\u8a18\u3067\u30bd\u30fc\u30c8\uff1a
IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8A08\uFF1A IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8a08\uff1a
IngestMessagePanel.totalMessagesNameVal.text=- IngestMessagePanel.totalMessagesNameVal.text=-
IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30E6\u30CB\u30FC\u30AF\uFF1A IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30e6\u30cb\u30fc\u30af\uff1a
IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestMessagePanel.totalUniqueMessagesNameVal.text=-
IngestMessagesToolbar.customizeButton.toolTipText=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E1\u30C3\u30BB\u30FC\u30B8 IngestMessagesToolbar.customizeButton.toolTipText=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e1\u30c3\u30bb\u30fc\u30b8
IngestMessageTopComponent.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 IngestMessageTopComponent.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9
IngestMessageTopComponent.displayReport.option.GenRpt=\u30EC\u30DD\u30FC\u30C8\u751F\u6210 IngestMessageTopComponent.displayReport.option.GenRpt=\u30ec\u30dd\u30fc\u30c8\u751f\u6210
IngestMessageTopComponent.displayReport.option.OK=OK IngestMessageTopComponent.displayReport.option.OK=OK
IngestMessageTopComponent.initComponents.name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 IngestMessageTopComponent.initComponents.name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9
IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30EC\u30DD\u30FC\u30C8 IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30ec\u30dd\u30fc\u30c8
IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30C7\u30A3\u30B9\u30AF{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3\u306E\u305F\u3081\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u4E2D\u6B62\u3057\u307E\u3059\u3002\ IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3\u306e\u305f\u3081\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u4e2d\u6b62\u3057\u307e\u3059\u3002\
\u30B1\u30FC\u30B9\u30C9\u30E9\u30A4\u30D6\u306B\u6700\u4F4E1GB\u306E\u7A7A\u304D\u9818\u57DF\u304C\u3042\u308B\u306E\u3092\u78BA\u8A8D\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u518D\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 \u30b1\u30fc\u30b9\u30c9\u30e9\u30a4\u30d6\u306b\u6700\u4f4e1GB\u306e\u7a7a\u304d\u9818\u57df\u304c\u3042\u308b\u306e\u3092\u78ba\u8a8d\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u518d\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002
IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u304C\u4E2D\u6B62\u3055\u308C\u307E\u3057\u305F\u30FC{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3 IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30B5\u30A4\u30BA\uFF1A IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30b5\u30a4\u30ba\uff1a
IngestScheduler.FileSched.toString.curDirs.text=\ OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
CurDirs(stack), \u30B5\u30A4\u30BA\uFF1A IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30a8\u30e9\u30fc\uff1a\
IngestScheduler.FileSched.toString.curFiles.text=\
CurFiles, \u30B5\u30A4\u30BA\uFF1A
IngestScheduler.FileSched.toString.rootDirs.text=\
RootDirs(sorted), \u30B5\u30A4\u30BA\uFF1A
OpenIDE-Module-Name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8
IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30A8\u30E9\u30FC\uFF1A\
\ \
{0} {0}
IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uFF11\u3064\u307E\u305F\u306F\u8907\u6570\u306E\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u30B9\u30BF\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u306F\u30AD\u30E3\u30F3\u30BB\u30EB\u3055\u308C\u307E\u3057\u305F\u3002 IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uff11\u3064\u307e\u305f\u306f\u8907\u6570\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u30b9\u30bf\u30fc\u30c8\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\u3002\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u306f\u30ad\u30e3\u30f3\u30bb\u30eb\u3055\u308c\u307e\u3057\u305f\u3002
IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305F\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u7121\u52B9\u5316\u3059\u308B\u304B\u30A8\u30E9\u30FC\u3092\u89E3\u6C7A\u3057\u3001\u305D\u306E\u5F8C\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u53F3\u30AF\u30EA\u30C3\u30AF\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u5B9F\u884C\u3092\u9078\u629E\u3057\u3066\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u30EA\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305f\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u7121\u52b9\u5316\u3059\u308b\u304b\u30a8\u30e9\u30fc\u3092\u89e3\u6c7a\u3057\u3001\u305d\u306e\u5f8c\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u53f3\u30af\u30ea\u30c3\u30af\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u5b9f\u884c\u3092\u9078\u629e\u3057\u3066\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u30ea\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002
IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u5931\u6557 IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u5931\u6557

View File

@ -18,16 +18,18 @@
*/ */
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
import org.netbeans.api.progress.ProgressHandle;
/** /**
* Used by data source ingest modules to report progress. * Used by data source ingest modules to report progress.
*/ */
public class DataSourceIngestModuleProgress { public class DataSourceIngestModuleProgress {
private final IngestJob ingestJob; private final ProgressHandle progress;
private final String moduleDisplayName; private final String moduleDisplayName;
DataSourceIngestModuleProgress(IngestJob ingestJob, String moduleDisplayName) { DataSourceIngestModuleProgress(ProgressHandle progress, String moduleDisplayName) {
this.ingestJob = ingestJob; this.progress = progress;
this.moduleDisplayName = moduleDisplayName; this.moduleDisplayName = moduleDisplayName;
} }
@ -40,7 +42,7 @@ public class DataSourceIngestModuleProgress {
* data source. * data source.
*/ */
public void switchToDeterminate(int workUnits) { public void switchToDeterminate(int workUnits) {
ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits); progress.switchToDeterminate(workUnits);
} }
/** /**
@ -48,7 +50,7 @@ public class DataSourceIngestModuleProgress {
* the total work units to process the data source is unknown. * the total work units to process the data source is unknown.
*/ */
public void switchToIndeterminate() { public void switchToIndeterminate() {
ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate(); progress.switchToIndeterminate();
} }
/** /**
@ -58,6 +60,6 @@ public class DataSourceIngestModuleProgress {
* @param workUnits Number of work units performed so far by the module. * @param workUnits Number of work units performed so far by the module.
*/ */
public void progress(int workUnits) { public void progress(int workUnits) {
ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits); progress.progress(this.moduleDisplayName, workUnits);
} }
} }

View File

@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.netbeans.api.progress.ProgressHandle;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
/** /**
@ -30,12 +31,12 @@ import org.sleuthkit.datamodel.Content;
*/ */
final class DataSourceIngestPipeline { final class DataSourceIngestPipeline {
private final IngestJob job; private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>(); private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
DataSourceIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) { DataSourceIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.job = task; this.context = context;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
} }
@ -50,7 +51,6 @@ final class DataSourceIngestPipeline {
for (IngestModuleTemplate template : moduleTemplates) { for (IngestModuleTemplate template : moduleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) { if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try { try {
module.startUp(context); module.startUp(context);
modulesByClass.put(module.getClassName(), module); modulesByClass.put(module.getClassName(), module);
@ -75,26 +75,26 @@ final class DataSourceIngestPipeline {
return errors; return errors;
} }
List<IngestModuleError> process() { List<IngestModuleError> process(Content dataSource, ProgressHandle progress) {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) { for (DataSourceIngestModuleDecorator module : this.modules) {
try { try {
module.process(job.getDataSource(), new DataSourceIngestModuleProgress(job, module.getDisplayName())); module.process(dataSource, new DataSourceIngestModuleProgress(progress, module.getDisplayName()));
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }
if (job.isCancelled()) { if (context.isJobCancelled()) {
break; break;
} }
} }
return errors; return errors;
} }
List<IngestModuleError> shutDown(boolean ingestJobCancelled) { List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) { for (DataSourceIngestModuleDecorator module : this.modules) {
try { try {
module.shutDown(ingestJobCancelled); module.shutDown(context.isJobCancelled());
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }

View File

@ -0,0 +1,40 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import org.sleuthkit.datamodel.Content;
final class DataSourceIngestTask extends IngestTask {
private final Content dataSource;
DataSourceIngestTask(IngestJob ingestJob, Content dataSource) {
super(ingestJob);
this.dataSource = dataSource;
}
Content getDataSource() {
return dataSource;
}
@Override
void execute() throws InterruptedException {
getIngestJob().process(dataSource);
}
}

View File

@ -30,12 +30,12 @@ import org.sleuthkit.datamodel.AbstractFile;
*/ */
final class FileIngestPipeline { final class FileIngestPipeline {
private final IngestJob job; private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private List<FileIngestModuleDecorator> modules = new ArrayList<>(); private List<FileIngestModuleDecorator> modules = new ArrayList<>();
FileIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) { FileIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.job = task; this.context = context;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
} }
@ -50,7 +50,6 @@ final class FileIngestPipeline {
for (IngestModuleTemplate template : moduleTemplates) { for (IngestModuleTemplate template : moduleTemplates) {
if (template.isFileIngestModuleTemplate()) { if (template.isFileIngestModuleTemplate()) {
FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try { try {
module.startUp(context); module.startUp(context);
modulesByClass.put(module.getClassName(), module); modulesByClass.put(module.getClassName(), module);
@ -83,22 +82,22 @@ final class FileIngestPipeline {
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }
if (job.isCancelled()) { if (context.isJobCancelled()) {
break; break;
} }
} }
file.close(); file.close();
if (!job.isCancelled()) { if (!context.isJobCancelled()) {
IngestManager.fireFileIngestDone(file.getId()); IngestManager.getInstance().fireFileIngestDone(file.getId());
} }
return errors; return errors;
} }
List<IngestModuleError> shutDown(boolean ingestJobCancelled) { List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) { for (FileIngestModuleDecorator module : this.modules) {
try { try {
module.shutDown(ingestJobCancelled); module.shutDown(context.isJobCancelled());
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }

View File

@ -0,0 +1,69 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.Objects;
import org.sleuthkit.datamodel.AbstractFile;
final class FileIngestTask extends IngestTask {
private final AbstractFile file;
FileIngestTask(IngestJob job, AbstractFile file) {
super(job);
this.file = file;
}
AbstractFile getFile() {
return file;
}
@Override
void execute() throws InterruptedException {
getIngestJob().process(file);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
FileIngestTask other = (FileIngestTask) obj;
IngestJob job = getIngestJob();
IngestJob otherJob = other.getIngestJob();
if (job != otherJob && (job == null || !job.equals(otherJob))) {
return false;
}
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
return false;
}
return true;
}
@Override
public int hashCode() {
int hash = 5;
hash = 47 * hash + Objects.hashCode(getIngestJob());
hash = 47 * hash + Objects.hashCode(this.file);
return hash;
}
}

View File

@ -0,0 +1,100 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
* Get counts of ingestable files/dirs for the content input source.
*
* Note, also includes counts of all unalloc children files (for the fs, image,
* volume) even if ingest didn't ask for them
*/
final class GetFilesCountVisitor extends ContentVisitor.Default<Long> {
private static final Logger logger = Logger.getLogger(GetFilesCountVisitor.class.getName());
@Override
public Long visit(FileSystem fs) {
//recursion stop here
//case of a real fs, query all files for it
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
StringBuilder queryB = new StringBuilder();
queryB.append("( (fs_obj_id = ").append(fs.getId()); //NON-NLS
//queryB.append(") OR (fs_obj_id = NULL) )");
queryB.append(") )");
queryB.append(" AND ( (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS
queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS
queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_VIRT.getValue()); //NON-NLS
queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS
queryB.append(") )");
//queryB.append( "AND (type = ");
//queryB.append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.getFileType());
//queryB.append(")");
try {
final String query = queryB.toString();
logger.log(Level.INFO, "Executing count files query: {0}", query); //NON-NLS
return sc.countFilesWhere(query);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); //NON-NLS
return 0L;
}
}
@Override
public Long visit(LayoutFile lf) {
//recursion stop here
//case of LayoutFile child of Image or Volume
return 1L;
}
private long getCountFromChildren(Content content) {
long count = 0;
try {
List<Content> children = content.getChildren();
if (children.size() > 0) {
for (Content child : children) {
count += child.accept(this);
}
} else {
count = 1;
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not get count of objects from children to get num of total files to be ingested", ex); //NON-NLS
}
return count;
}
@Override
protected Long defaultVisit(Content cntnt) {
//recurse assuming this is image/vs/volume
//recursion stops at fs or unalloc file
return getCountFromChildren(cntnt);
}
}

View File

@ -0,0 +1,88 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.VirtualDirectory;
/**
* Finds top level objects such as file system root directories, layout
* files and virtual directories.
*/
final class GetRootDirectoryVisitor extends GetFilesContentVisitor {
@Override
public Collection<AbstractFile> visit(VirtualDirectory ld) {
//case when we hit a layout directoryor local file container, not under a real FS
//or when root virt dir is scheduled
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(ld);
return ret;
}
@Override
public Collection<AbstractFile> visit(LayoutFile lf) {
//case when we hit a layout file, not under a real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(lf);
return ret;
}
@Override
public Collection<AbstractFile> visit(Directory drctr) {
//we hit a real directory, a child of real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(drctr);
return ret;
}
@Override
public Collection<AbstractFile> visit(FileSystem fs) {
return getAllFromChildren(fs);
}
@Override
public Collection<AbstractFile> visit(File file) {
//can have derived files
return getAllFromChildren(file);
}
@Override
public Collection<AbstractFile> visit(DerivedFile derivedFile) {
//can have derived files
//TODO test this and overall scheduler with derived files
return getAllFromChildren(derivedFile);
}
@Override
public Collection<AbstractFile> visit(LocalFile localFile) {
//can have local files
//TODO test this and overall scheduler with local files
return getAllFromChildren(localFile);
}
}

View File

@ -19,83 +19,111 @@
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory; import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable; import org.openide.util.Cancellable;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
/**
* Encapsulates a data source and the ingest module pipelines to be used to
* ingest the data source.
*/
final class IngestJob { final class IngestJob {
private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private final long id; private final long id;
private final Content dataSource; private final Content rootDataSource;
private final List<IngestModuleTemplate> ingestModuleTemplates; private final List<IngestModuleTemplate> ingestModuleTemplates;
private final boolean processUnallocatedSpace; private final boolean processUnallocatedSpace;
private final HashMap<Long, FileIngestPipeline> fileIngestPipelines = new HashMap<>(); private final LinkedBlockingQueue<DataSourceIngestPipeline> dataSourceIngestPipelines = new LinkedBlockingQueue<>();
private final HashMap<Long, DataSourceIngestPipeline> dataSourceIngestPipelines = new HashMap<>(); private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelines = new LinkedBlockingQueue<>();
private final IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); private long estimatedFilesToProcess = 0L; // Guarded by this
private FileIngestPipeline initialFileIngestPipeline = null; private long processedFiles = 0L; // Guarded by this
private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; private ProgressHandle dataSourceTasksProgress;
private ProgressHandle dataSourceTaskProgress;
private ProgressHandle fileTasksProgress; private ProgressHandle fileTasksProgress;
int totalEnqueuedFiles = 0; private volatile boolean cancelled = false;
private int processedFiles = 0;
private volatile boolean cancelled;
IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) { IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) {
this.id = id; this.id = id;
this.dataSource = dataSource; this.rootDataSource = dataSource;
this.ingestModuleTemplates = ingestModuleTemplates; this.ingestModuleTemplates = ingestModuleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace; this.processUnallocatedSpace = processUnallocatedSpace;
this.cancelled = false;
} }
long getId() { long getId() {
return id; return id;
} }
Content getDataSource() {
return dataSource;
}
boolean shouldProcessUnallocatedSpace() { boolean shouldProcessUnallocatedSpace() {
return processUnallocatedSpace; return processUnallocatedSpace;
} }
synchronized List<IngestModuleError> startUpIngestPipelines() { List<IngestModuleError> startUp() throws InterruptedException {
startDataSourceIngestProgressBar(); List<IngestModuleError> errors = startUpIngestPipelines();
startFileIngestProgressBar(); if (errors.isEmpty()) {
return startUpInitialIngestPipelines(); startFileIngestProgressBar();
startDataSourceIngestProgressBar();
}
return errors;
}
private List<IngestModuleError> startUpIngestPipelines() throws InterruptedException {
IngestJobContext context = new IngestJobContext(this);
List<IngestModuleError> errors = new ArrayList<>();
int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
dataSourceIngestPipelines.put(pipeline);
if (!errors.isEmpty()) {
// No need to accumulate presumably redundant errors.
break;
}
}
maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
FileIngestPipeline pipeline = new FileIngestPipeline(context, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
fileIngestPipelines.put(pipeline);
if (!errors.isEmpty()) {
// No need to accumulate presumably redundant errors.
break;
}
}
logIngestModuleErrors(errors);
return errors; // Returned so UI can report to user.
} }
private void startDataSourceIngestProgressBar() { private void startDataSourceIngestProgressBar() {
final String displayName = NbBundle final String displayName = NbBundle.getMessage(this.getClass(),
.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", this.dataSource.getName()); "IngestJob.progress.dataSourceIngest.displayName",
dataSourceTaskProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { rootDataSource.getName());
dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
if (dataSourceTaskProgress != null) { if (dataSourceTasksProgress != null) {
dataSourceTaskProgress.setDisplayName(NbBundle.getMessage(this.getClass(), dataSourceTasksProgress.setDisplayName(
NbBundle.getMessage(this.getClass(),
"IngestJob.progress.cancelling", "IngestJob.progress.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); IngestJob.this.cancel();
return true; return true;
} }
}); });
dataSourceTaskProgress.start(); dataSourceTasksProgress.start();
dataSourceTaskProgress.switchToIndeterminate(); dataSourceTasksProgress.switchToIndeterminate();
} }
private void startFileIngestProgressBar() { private void startFileIngestProgressBar() {
final String displayName = NbBundle final String displayName = NbBundle.getMessage(this.getClass(),
.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); "IngestJob.progress.fileIngest.displayName",
rootDataSource.getName());
fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
@ -104,124 +132,84 @@ final class IngestJob {
NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); IngestJob.this.cancel();
return true; return true;
} }
}); });
estimatedFilesToProcess = rootDataSource.accept(new GetFilesCountVisitor());
fileTasksProgress.start(); fileTasksProgress.start();
fileTasksProgress.switchToIndeterminate(); fileTasksProgress.switchToDeterminate((int) estimatedFilesToProcess);
totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
} }
private List<IngestModuleError> startUpInitialIngestPipelines() { void process(Content dataSource) throws InterruptedException {
// Create a per thread instance of each pipeline type right now to make // If the job is not cancelled, complete the task, otherwise just flush
// (reasonably) sure that the ingest modules can be started. // it. In either case, the task counter needs to be decremented and the
initialDataSourceIngestPipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); // shut down check needs to occur.
initialFileIngestPipeline = new FileIngestPipeline(this, ingestModuleTemplates); if (!isCancelled()) {
List<IngestModuleError> errors = new ArrayList<>();
DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take();
errors.addAll(pipeline.process(dataSource, dataSourceTasksProgress));
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
dataSourceIngestPipelines.put(pipeline);
}
}
void process(AbstractFile file) throws InterruptedException {
// If the job is not cancelled, complete the task, otherwise just flush
// it. In either case, the task counter needs to be decremented and the
// shut down check needs to occur.
if (!isCancelled()) {
List<IngestModuleError> errors = new ArrayList<>();
synchronized (this) {
++processedFiles;
if (processedFiles <= estimatedFilesToProcess) {
fileTasksProgress.progress(file.getName(), (int) processedFiles);
} else {
fileTasksProgress.progress(file.getName(), (int) estimatedFilesToProcess);
}
}
FileIngestPipeline pipeline = fileIngestPipelines.take();
errors.addAll(pipeline.process(file));
fileIngestPipelines.put(pipeline);
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
}
}
void shutDown() {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(initialDataSourceIngestPipeline.startUp()); while (!dataSourceIngestPipelines.isEmpty()) {
errors.addAll(initialFileIngestPipeline.startUp()); DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll();
return errors; errors.addAll(pipeline.shutDown());
}
while (!fileIngestPipelines.isEmpty()) {
FileIngestPipeline pipeline = fileIngestPipelines.poll();
errors.addAll(pipeline.shutDown());
}
fileTasksProgress.finish();
dataSourceTasksProgress.finish();
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
} }
synchronized DataSourceIngestPipeline getDataSourceIngestPipelineForThread(long threadId) { private void logIngestModuleErrors(List<IngestModuleError> errors) {
DataSourceIngestPipeline pipeline; for (IngestModuleError error : errors) {
if (initialDataSourceIngestPipeline != null) { logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError());
pipeline = initialDataSourceIngestPipeline;
initialDataSourceIngestPipeline = null;
dataSourceIngestPipelines.put(threadId, pipeline);
} else if (!dataSourceIngestPipelines.containsKey(threadId)) {
pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates);
pipeline.startUp();
dataSourceIngestPipelines.put(threadId, pipeline);
} else {
pipeline = dataSourceIngestPipelines.get(threadId);
} }
return pipeline;
}
synchronized FileIngestPipeline getFileIngestPipelineForThread(long threadId) {
FileIngestPipeline pipeline;
if (initialFileIngestPipeline != null) {
pipeline = initialFileIngestPipeline;
initialFileIngestPipeline = null;
fileIngestPipelines.put(threadId, pipeline);
} else if (!fileIngestPipelines.containsKey(threadId)) {
pipeline = new FileIngestPipeline(this, ingestModuleTemplates);
pipeline.startUp();
fileIngestPipelines.put(threadId, pipeline);
} else {
pipeline = fileIngestPipelines.get(threadId);
}
return pipeline;
}
synchronized List<IngestModuleError> releaseIngestPipelinesForThread(long threadId) {
List<IngestModuleError> errors = new ArrayList<>();
DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId);
if (dataSourceIngestPipeline != null) {
errors.addAll(dataSourceIngestPipeline.shutDown(cancelled));
dataSourceIngestPipelines.remove(threadId);
}
if (initialDataSourceIngestPipeline == null && dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) {
dataSourceTaskProgress.finish();
dataSourceTaskProgress = null;
}
FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId);
if (fileIngestPipeline != null) {
errors.addAll(fileIngestPipeline.shutDown(cancelled));
fileIngestPipelines.remove(threadId);
}
if (initialFileIngestPipeline == null && fileIngestPipelines.isEmpty() && fileTasksProgress != null) {
fileTasksProgress.finish();
fileTasksProgress = null;
}
return errors;
}
synchronized boolean areIngestPipelinesShutDown() {
return (initialDataSourceIngestPipeline == null
&& dataSourceIngestPipelines.isEmpty()
&& initialFileIngestPipeline == null
&& fileIngestPipelines.isEmpty());
}
synchronized ProgressHandle getDataSourceTaskProgressBar() {
return this.dataSourceTaskProgress;
}
synchronized void updateFileTasksProgressBar(String currentFileName) {
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
totalEnqueuedFiles = newTotalEnqueuedFiles + 1;
fileTasksProgress.switchToIndeterminate();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
}
if (processedFiles < totalEnqueuedFiles) {
++processedFiles;
}
fileTasksProgress.progress(currentFileName, processedFiles);
}
synchronized void cancel() {
if (initialDataSourceIngestPipeline != null) {
initialDataSourceIngestPipeline.shutDown(true);
initialDataSourceIngestPipeline = null;
}
if (initialFileIngestPipeline != null) {
initialFileIngestPipeline.shutDown(true);
initialFileIngestPipeline = null;
}
cancelled = true;
} }
boolean isCancelled() { boolean isCancelled() {
return cancelled; return cancelled;
} }
void cancel() {
cancelled = true;
fileTasksProgress.finish();
dataSourceTasksProgress.finish();
IngestManager.getInstance().fireIngestJobCancelled(id);
}
} }

View File

@ -41,14 +41,14 @@ import org.sleuthkit.datamodel.Content;
* for a particular context and for launching ingest jobs that process one or * for a particular context and for launching ingest jobs that process one or
* more data sources using the ingest job configuration. * more data sources using the ingest job configuration.
*/ */
public final class IngestJobLauncher { public final class IngestJobConfigurator {
private static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; //NON-NLS private static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; //NON-NLS
private static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; //NON-NLS private static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; //NON-NLS
private static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; //NON-NLS private static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; //NON-NLS
private static final String MODULE_SETTINGS_FOLDER_PATH = new StringBuilder(PlatformUtil.getUserConfigDirectory()).append(File.separator).append("IngestModuleSettings").toString(); //NON-NLS private static final String MODULE_SETTINGS_FOLDER_PATH = new StringBuilder(PlatformUtil.getUserConfigDirectory()).append(File.separator).append("IngestModuleSettings").toString(); //NON-NLS
private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS
private static final Logger logger = Logger.getLogger(IngestJobLauncher.class.getName()); private static final Logger logger = Logger.getLogger(IngestJobConfigurator.class.getName());
private final String launcherContext; private final String launcherContext;
private String moduleSettingsFolderForContext = null; private String moduleSettingsFolderForContext = null;
private final List<String> warnings = new ArrayList<>(); private final List<String> warnings = new ArrayList<>();
@ -61,7 +61,7 @@ public final class IngestJobLauncher {
* *
* @param launcherContext The context identifier. * @param launcherContext The context identifier.
*/ */
public IngestJobLauncher(String launcherContext) { public IngestJobConfigurator(String launcherContext) {
this.launcherContext = launcherContext; this.launcherContext = launcherContext;
createModuleSettingsFolderForContext(); createModuleSettingsFolderForContext();

View File

@ -60,7 +60,7 @@ public final class IngestJobContext {
*/ */
public void addFiles(List<AbstractFile> files) { public void addFiles(List<AbstractFile> files) {
for (AbstractFile file : files) { for (AbstractFile file : files) {
IngestManager.getInstance().addFileToIngestJob(ingestJob.getId(), file); IngestScheduler.getInstance().addFileToIngestJob(ingestJob, file);
} }
} }
} }

View File

@ -20,10 +20,10 @@ package org.sleuthkit.autopsy.ingest;
import java.beans.PropertyChangeListener; import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport; import java.beans.PropertyChangeSupport;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.CancellationException; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
@ -36,72 +36,148 @@ import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable; import org.openide.util.Cancellable;
import org.openide.util.NbPreferences; import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
import java.util.prefs.Preferences; import java.util.prefs.Preferences;
import javax.swing.JOptionPane; import javax.swing.JOptionPane;
import javax.swing.SwingWorker;
import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask;
/** /**
* Manages the execution of ingest jobs. * Manages the execution of ingest jobs.
*/ */
public class IngestManager { public class IngestManager {
private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1;
private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS
private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1;
private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16;
private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2;
private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class);
private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class);
private static final IngestManager instance = new IngestManager(); private static final IngestManager instance = new IngestManager();
private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final IngestScheduler scheduler = IngestScheduler.getInstance(); private final IngestScheduler scheduler = IngestScheduler.getInstance();
private final IngestMonitor ingestMonitor = new IngestMonitor(); private final IngestMonitor ingestMonitor = new IngestMonitor();
private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS);
private final ExecutorService fireEventTasksExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor();
private final ConcurrentHashMap<Long, IngestJob> ingestJobs = new ConcurrentHashMap<>(1, 0.9f, 4); // Maps job ids to jobs. private final AtomicLong nextThreadId = new AtomicLong(0L);
private final ConcurrentHashMap<Long, Future<?>> ingestTasks = new ConcurrentHashMap<>(); // Maps task ids to task cancellation handles. Guarded by this. private final ConcurrentHashMap<Long, Future<Void>> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final AtomicLong ingestJobId = new AtomicLong(0L); private final ConcurrentHashMap<Long, Future<?>> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final AtomicLong ingestTaskId = new AtomicLong(0L); private final ConcurrentHashMap<Long, Future<?>> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private volatile IngestMessageTopComponent ingestMessageBox; private volatile IngestMessageTopComponent ingestMessageBox;
/** /**
* Gets the IngestManager singleton, creating it if necessary. * Gets the ingest manager.
* *
* @returns The IngestManager singleton. * @returns A singleton IngestManager object.
*/ */
public static IngestManager getInstance() { public static IngestManager getInstance() {
return instance; return instance;
} }
private IngestManager() {
}
/** /**
* Signals to the ingest manager that it can go find the top component for * Starts the ingest monitor and the data source ingest and file ingest
* the ingest messages in box. Called by the custom installer for this * threads.
* package once the window system is initialized.
*/ */
void initIngestMessageInbox() { private IngestManager() {
if (this.ingestMessageBox == null) { startDataSourceIngestThread();
this.ingestMessageBox = IngestMessageTopComponent.findInstance(); int numberOfFileIngestThreads = getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfFileIngestThreads; ++i) {
startFileIngestThread();
} }
} }
/**
* Signals to the ingest manager that it can go about finding the top
* component for the ingest messages in box. Called by the custom installer
* for this package once the window system is initialized.
*/
void initIngestMessageInbox() {
if (ingestMessageBox == null) {
ingestMessageBox = IngestMessageTopComponent.findInstance();
}
}
/**
* Gets the maximum number of data source ingest threads the ingest manager
* will use.
*/
public static int getMaxNumberOfDataSourceIngestThreads() {
return MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS;
}
/**
* Gets the maximum number of file ingest threads the ingest manager will
* use.
*/
public static int getMaxNumberOfFileIngestThreads() {
return MAX_NUMBER_OF_FILE_INGEST_THREADS;
}
/**
* Gets the number of file ingest threads the ingest manager will use.
*/
public synchronized static int getNumberOfFileIngestThreads() { public synchronized static int getNumberOfFileIngestThreads() {
return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS);
} }
/**
* Changes the number of file ingest threads the ingest manager will use to
* no more than MAX_NUMBER_OF_FILE_INGEST_THREADS and no less than
* MIN_NUMBER_OF_FILE_INGEST_THREADS. Out of range requests are converted to
* requests for DEFAULT_NUMBER_OF_FILE_INGEST_THREADS.
*
* @param numberOfThreads The desired number of file ingest threads.
*/
public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) { public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) {
if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS if ((numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) {
|| numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) {
numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS;
} }
userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads);
if (instance.fileIngestThreads.size() != numberOfThreads) {
if (instance.fileIngestThreads.size() > numberOfThreads) {
Long[] threadIds = instance.fileIngestThreads.keySet().toArray(new Long[instance.fileIngestThreads.size()]);
int numberOfThreadsToCancel = instance.fileIngestThreads.size() - numberOfThreads;
for (int i = 0; i < numberOfThreadsToCancel; ++i) {
instance.cancelFileIngestThread(threadIds[i]);
}
} else if (instance.fileIngestThreads.size() < numberOfThreads) {
int numberOfThreadsToAdd = numberOfThreads - instance.fileIngestThreads.size();
for (int i = 0; i < numberOfThreadsToAdd; ++i) {
instance.startFileIngestThread();
}
}
}
}
/**
* Submits a DataSourceIngestThread Runnable to the data source ingest
* thread pool.
*/
private void startDataSourceIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = dataSourceIngestThreadPool.submit(new ExecuteIngestTasksThread(scheduler.getDataSourceIngestTaskQueue()));
dataSourceIngestThreads.put(threadId, handle);
}
/**
* Submits a DataSourceIngestThread Runnable to the data source ingest
* thread pool.
*/
private void startFileIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = fileIngestThreadPool.submit(new ExecuteIngestTasksThread(scheduler.getFileIngestTaskQueue()));
fileIngestThreads.put(threadId, handle);
}
/**
* Cancels a DataSourceIngestThread Runnable in the file ingest thread pool.
*/
private void cancelFileIngestThread(long threadId) {
Future<?> handle = fileIngestThreads.remove(threadId);
handle.cancel(true);
} }
synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) { synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
@ -109,9 +185,9 @@ public class IngestManager {
ingestMessageBox.clearMessages(); ingestMessageBox.clearMessages();
} }
long taskId = ingestTaskId.incrementAndGet(); long taskId = nextThreadId.incrementAndGet();
Future<?> task = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); Future<Void> task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
ingestTasks.put(taskId, task); startIngestJobThreads.put(taskId, task);
if (ingestMessageBox != null) { if (ingestMessageBox != null) {
ingestMessageBox.restoreMessages(); ingestMessageBox.restoreMessages();
@ -121,21 +197,31 @@ public class IngestManager {
/** /**
* Test if any ingest jobs are in progress. * Test if any ingest jobs are in progress.
* *
* @return True if any ingest jobs are in progress, false otherwise * @return True if any ingest jobs are in progress, false otherwise.
*/ */
public boolean isIngestRunning() { public boolean isIngestRunning() {
return (ingestJobs.isEmpty() == false); return scheduler.ingestJobsAreRunning();
} }
void addFileToIngestJob(long ingestJobId, AbstractFile file) { public void cancelAllIngestJobs() {
IngestJob job = ingestJobs.get(ingestJobId); // Stop creating new ingest jobs.
if (job != null) { for (Future<Void> handle : startIngestJobThreads.values()) {
scheduler.getFileIngestScheduler().queueFile(job, file); handle.cancel(true);
try {
// Blocks until the job starting thread responds. The thread
// removes itself from this collection, which does not disrupt
// this loop since the collection is a ConcurrentHashMap.
handle.get();
} catch (InterruptedException | ExecutionException ex) {
// This should never happen, something is awry, but everything
// should be o.k. anyway.
logger.log(Level.SEVERE, "Unexpected thread interrupt", ex);
}
} }
}
void cancelIngestJobs() { // Cancel all the jobs already created. This will make the the ingest
new IngestCancellationWorker().execute(); // threads flush out any lingering ingest tasks without processing them.
scheduler.cancelAllIngestJobs();
} }
/** /**
@ -144,20 +230,21 @@ public class IngestManager {
public enum IngestEvent { public enum IngestEvent {
/** /**
* Property change event fired when an ingest job is started. The ingest * Property change event fired when an ingest job is started. The old
* job id is in old value field of the PropertyChangeEvent object. * value of the PropertyChangeEvent object is set to the ingest job id,
* and the new value is set to null.
*/ */
INGEST_JOB_STARTED, INGEST_JOB_STARTED,
/** /**
* Property change event fired when an ingest job is completed. The * Property change event fired when an ingest job is completed. The old
* ingest job id is in old value field of the PropertyChangeEvent * value of the PropertyChangeEvent object is set to the ingest job id,
* object. * and the new value is set to null.
*/ */
INGEST_JOB_COMPLETED, INGEST_JOB_COMPLETED,
/** /**
* Property change event fired when an ingest job is canceled. The * Property change event fired when an ingest job is canceled. The old
* ingest job id is in old value field of the PropertyChangeEvent * value of the PropertyChangeEvent object is set to the ingest job id,
* object. * and the new value is set to null.
*/ */
INGEST_JOB_CANCELLED, INGEST_JOB_CANCELLED,
/** /**
@ -182,79 +269,122 @@ public class IngestManager {
}; };
/** /**
* Add property change listener to listen to ingest events. * Add an ingest job event property change listener.
* *
* @param listener PropertyChangeListener to register * @param listener The PropertyChangeListener to register.
*/
public void addIngestJobEventListener(final PropertyChangeListener listener) {
ingestJobEventPublisher.addPropertyChangeListener(listener);
}
/**
* Remove an ingest job event property change listener.
*
* @param listener The PropertyChangeListener to unregister.
*/
public void removeIngestJobEventListener(final PropertyChangeListener listener) {
ingestJobEventPublisher.removePropertyChangeListener(listener);
}
/**
* Add an ingest module event property change listener.
*
* @param listener The PropertyChangeListener to register.
*/
public void addIngestModuleEventListener(final PropertyChangeListener listener) {
ingestModuleEventPublisher.addPropertyChangeListener(listener);
}
/**
* Remove an ingest module event property change listener.
*
* @param listener The PropertyChangeListener to unregister.
*/
public void removeIngestModuleEventListener(final PropertyChangeListener listener) {
ingestModuleEventPublisher.removePropertyChangeListener(listener);
}
/**
* Add an ingest module event property change listener.
*
* @deprecated
* @param listener The PropertyChangeListener to register.
*/ */
public static void addPropertyChangeListener(final PropertyChangeListener listener) { public static void addPropertyChangeListener(final PropertyChangeListener listener) {
pcs.addPropertyChangeListener(listener); instance.ingestModuleEventPublisher.addPropertyChangeListener(listener);
} }
/**
* Remove an ingest module event property change listener.
*
* @deprecated
* @param listener The PropertyChangeListener to unregister.
*/
public static void removePropertyChangeListener(final PropertyChangeListener listener) { public static void removePropertyChangeListener(final PropertyChangeListener listener) {
pcs.removePropertyChangeListener(listener); instance.ingestModuleEventPublisher.removePropertyChangeListener(listener);
}
static void fireIngestJobEvent(String eventType, long jobId) {
try {
pcs.firePropertyChange(eventType, jobId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event when file is done with a pipeline run * Fire an ingest event signifying an ingest job started.
* *
* @param fileId ID of file that is done * @param ingestJobId The ingest job id.
*/ */
static void fireFileIngestDone(long fileId) { void fireIngestJobStarted(long ingestJobId) {
try { fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_STARTED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event for ModuleDataEvent (when modules post data to blackboard, * Fire an ingest event signifying an ingest job finished.
* etc.)
* *
* @param moduleDataEvent * @param ingestJobId The ingest job id.
*/ */
static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { void fireIngestJobCompleted(long ingestJobId) {
try { fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event for ModuleContentChanged (when modules create new content that * Fire an ingest event signifying an ingest job was canceled.
* needs to be analyzed)
* *
* @param moduleContentEvent * @param ingestJobId The ingest job id.
*/ */
static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { void fireIngestJobCancelled(long ingestJobId) {
try { fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/**
* Fire an ingest event signifying the ingest of a file is completed.
*
* @param fileId The object id of file.
*/
void fireFileIngestDone(long fileId) {
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.FILE_DONE, fileId, null));
}
/**
* Fire an event signifying a blackboard post by an ingest module.
*
* @param moduleDataEvent A ModuleDataEvent with the details of the posting.
*/
void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) {
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.DATA, moduleDataEvent, null));
}
/**
* Fire an event signifying discovery of additional content by an ingest
* module.
*
* @param moduleDataEvent A ModuleContentEvent with the details of the new
* content.
*/
void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) {
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.CONTENT_CHANGED, moduleContentEvent, null));
}
/**
* Post a message to the ingest messages in box.
*
* @param message The message to be posted.
*/
void postIngestMessage(IngestMessage message) { void postIngestMessage(IngestMessage message) {
if (ingestMessageBox != null) { if (ingestMessageBox != null) {
ingestMessageBox.displayMessage(message); ingestMessageBox.displayMessage(message);
@ -262,11 +392,10 @@ public class IngestManager {
} }
/** /**
* Get free disk space of a drive where ingest data are written to That * Get the free disk space of the drive where to which ingest data is being
* drive is being monitored by IngestMonitor thread when ingest is running. * written, as reported by the ingest monitor.
* Use this method to get amount of free disk space anytime.
* *
* @return amount of disk space, -1 if unknown * @return Free disk space, -1 if unknown
*/ */
long getFreeDiskSpace() { long getFreeDiskSpace() {
if (ingestMonitor != null) { if (ingestMonitor != null) {
@ -276,40 +405,26 @@ public class IngestManager {
} }
} }
private void reportRunIngestModulesTaskDone(long taskId) { /**
ingestTasks.remove(taskId); * Creates ingest jobs.
*/
private class StartIngestJobsThread implements Callable<Void> {
List<Long> completedJobs = new ArrayList<>(); private final long threadId;
for (IngestJob job : ingestJobs.values()) {
job.releaseIngestPipelinesForThread(taskId);
if (job.areIngestPipelinesShutDown() == true) {
completedJobs.add(job.getId());
}
}
for (Long jobId : completedJobs) {
IngestJob job = ingestJobs.remove(jobId);
fireEventTasksExecutor.submit(new FireIngestJobEventTask(jobId, job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED : IngestEvent.INGEST_JOB_COMPLETED));
}
}
private class StartIngestJobsTask implements Runnable {
private final long id;
private final List<Content> dataSources; private final List<Content> dataSources;
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private final boolean processUnallocatedSpace; private final boolean processUnallocatedSpace;
private ProgressHandle progress; private ProgressHandle progress;
StartIngestJobsTask(long taskId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) { StartIngestJobsThread(long threadId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
this.id = taskId; this.threadId = threadId;
this.dataSources = dataSources; this.dataSources = dataSources;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace; this.processUnallocatedSpace = processUnallocatedSpace;
} }
@Override @Override
public void run() { public Void call() {
try { try {
final String displayName = NbBundle.getMessage(this.getClass(), final String displayName = NbBundle.getMessage(this.getClass(),
"IngestManager.StartIngestJobsTask.run.displayName"); "IngestManager.StartIngestJobsTask.run.displayName");
@ -321,33 +436,29 @@ public class IngestManager {
"IngestManager.StartIngestJobsTask.run.cancelling", "IngestManager.StartIngestJobsTask.run.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); cancelFileIngestThread(threadId);
return true; return true;
} }
}); });
progress.start(dataSources.size());
progress.start(2 * dataSources.size()); if (!ingestMonitor.isRunning()) {
int workUnitsCompleted = 0; ingestMonitor.start();
}
int dataSourceProcessed = 0;
for (Content dataSource : dataSources) { for (Content dataSource : dataSources) {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
break; break;
} }
// Create an ingest job. // Start an ingest job for the data source.
IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace); List<IngestModuleError> errors = scheduler.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace);
ingestJobs.put(ingestJob.getId(), ingestJob);
// Start at least one instance of each kind of ingest
// pipeline for this ingest job. This allows for an early out
// if the full ingest module lineup specified by the user
// cannot be started up.
List<IngestModuleError> errors = ingestJob.startUpIngestPipelines();
if (!errors.isEmpty()) { if (!errors.isEmpty()) {
// Report the error to the user. // Report the errors to the user. They have already been logged.
StringBuilder moduleStartUpErrors = new StringBuilder(); StringBuilder moduleStartUpErrors = new StringBuilder();
for (IngestModuleError error : errors) { for (IngestModuleError error : errors) {
String moduleName = error.getModuleDisplayName(); String moduleName = error.getModuleDisplayName();
logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); //NON-NLS
moduleStartUpErrors.append(moduleName); moduleStartUpErrors.append(moduleName);
moduleStartUpErrors.append(": "); moduleStartUpErrors.append(": ");
moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage());
@ -367,169 +478,75 @@ public class IngestManager {
JOptionPane.showMessageDialog(null, notifyMessage.toString(), JOptionPane.showMessageDialog(null, notifyMessage.toString(),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(),
"IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE);
// Jettison the ingest job and move on to the next one.
ingestJob.cancel();
ingestJobs.remove(ingestJob.getId());
break;
} }
progress.progress(++dataSourceProcessed);
// Queue the data source ingest tasks for the ingest job.
final String inputName = dataSource.getName();
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg1",
inputName), workUnitsCompleted);
scheduler.getDataSourceIngestScheduler().queueForIngest(ingestJob);
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg2",
inputName), ++workUnitsCompleted);
// Queue the file ingest tasks for the ingest job.
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg3",
inputName), workUnitsCompleted);
scheduler.getFileIngestScheduler().queueForIngest(ingestJob);
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg4",
inputName), ++workUnitsCompleted);
if (!Thread.currentThread().isInterrupted()) { if (!Thread.currentThread().isInterrupted()) {
if (!ingestMonitor.isRunning()) { break;
ingestMonitor.start();
}
long taskId = ingestTaskId.incrementAndGet();
Future<?> task = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
int numberOfFileTasksRequested = getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfFileTasksRequested; ++i) {
taskId = ingestTaskId.incrementAndGet();
task = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
}
fireEventTasksExecutor.submit(new FireIngestJobEventTask(ingestJob.getId(), IngestEvent.INGEST_JOB_STARTED));
} }
} }
} catch (Exception ex) {
String message = String.format("StartIngestJobsTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
MessageNotifyUtil.Message.error(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg"));
} finally { } finally {
progress.finish(); progress.finish();
ingestTasks.remove(id); startIngestJobThreads.remove(threadId);
return null;
} }
} }
} }
private class RunDataSourceIngestModulesTask implements Runnable { /**
* A consumer for an ingest task queue.
*/
private class ExecuteIngestTasksThread implements Runnable {
private final long id; private IngestTaskQueue tasks;
RunDataSourceIngestModulesTask(long taskId) { ExecuteIngestTasksThread(IngestTaskQueue tasks) {
id = taskId; this.tasks = tasks;
} }
@Override @Override
public void run() { public void run() {
try { while (true) {
IngestScheduler.DataSourceIngestScheduler scheduler = IngestScheduler.getInstance().getDataSourceIngestScheduler(); try {
IngestJob job = scheduler.getNextTask(); IngestTask task = tasks.getNextTask(); // Blocks.
while (job != null) { task.execute();
if (Thread.currentThread().isInterrupted()) { scheduler.ingestTaskIsCompleted(task);
break; } catch (InterruptedException ex) {
} break;
job.getDataSourceIngestPipelineForThread(id).process(); }
job = scheduler.getNextTask(); if (Thread.currentThread().isInterrupted()) {
break;
} }
} catch (Exception ex) {
String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
} }
} }
} }
private class RunFileSourceIngestModulesTask implements Runnable { /**
* Fires ingest events to ingest manager property change listeners.
*/
private static class FireIngestEventThread implements Runnable {
private final long id; private final PropertyChangeSupport publisher;
RunFileSourceIngestModulesTask(long taskId) {
id = taskId;
}
@Override
public void run() {
try {
IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler();
FileIngestTask task = fileScheduler.getNextTask();
while (task != null) {
if (Thread.currentThread().isInterrupted()) {
break;
}
IngestJob job = task.getJob();
job.updateFileTasksProgressBar(task.getFile().getName());
job.getFileIngestPipelineForThread(id).process(task.getFile());
task = fileScheduler.getNextTask();
}
} catch (Exception ex) {
String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
}
}
}
private class FireIngestJobEventTask implements Runnable {
private final long ingestJobId;
private final IngestEvent event; private final IngestEvent event;
private final Object oldValue;
private final Object newValue;
FireIngestJobEventTask(long ingestJobId, IngestEvent event) { FireIngestEventThread(PropertyChangeSupport publisher, IngestEvent event, Object oldValue, Object newValue) {
this.ingestJobId = ingestJobId; this.publisher = publisher;
this.event = event; this.event = event;
this.oldValue = oldValue;
this.newValue = newValue;
} }
@Override @Override
public void run() { public void run() {
fireIngestJobEvent(event.toString(), ingestJobId);
}
}
private class IngestCancellationWorker extends SwingWorker<Void, Void> {
@Override
protected Void doInBackground() throws Exception {
// First mark all of the ingest jobs as cancelled. This way the
// ingest modules will know they are being shut down due to
// cancellation when the cancelled run ingest module tasks release
// their pipelines.
for (IngestJob job : ingestJobs.values()) {
job.cancel();
}
for (Future<?> task : ingestTasks.values()) {
task.cancel(true);
}
// Jettision the remaining data source and file ingest tasks.
scheduler.getFileIngestScheduler().emptyQueues();
scheduler.getDataSourceIngestScheduler().emptyQueues();
return null;
}
@Override
protected void done() {
try { try {
super.get(); publisher.firePropertyChange(event.toString(), oldValue, newValue);
} catch (CancellationException | InterruptedException ex) { } catch (Exception e) {
} catch (Exception ex) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
logger.log(Level.SEVERE, "Error while cancelling ingest jobs", ex); //NON-NLS MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
} }
} }
} }

View File

@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content;
manager = IngestManager.getInstance(); manager = IngestManager.getInstance();
} }
try { try {
manager.cancelIngestJobs(); manager.cancelAllIngestJobs();
} finally { } finally {
//clear inbox //clear inbox
clearMessages(); clearMessages();

View File

@ -165,7 +165,7 @@ public final class IngestMonitor {
final String diskPath = root.getAbsolutePath(); final String diskPath = root.getAbsolutePath();
MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS
logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS
manager.cancelIngestJobs(); manager.cancelAllIngestJobs();
IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage( IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage(
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath), NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath),
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath))); NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath)));

1018
Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java Normal file → Executable file

File diff suppressed because it is too large Load Diff

View File

@ -95,7 +95,7 @@ public final class IngestServices {
* artifact data * artifact data
*/ */
public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
IngestManager.fireModuleDataEvent(moduleDataEvent); IngestManager.getInstance().fireIngestModuleDataEvent(moduleDataEvent);
} }
/** /**
@ -107,7 +107,7 @@ public final class IngestServices {
* changed * changed
*/ */
public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
IngestManager.fireModuleContentEvent(moduleContentEvent); IngestManager.getInstance().fireIngestModuleContentEvent(moduleContentEvent);
} }
/** /**

View File

@ -0,0 +1,34 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
abstract class IngestTask {
private final IngestJob job;
IngestTask(IngestJob job) {
this.job = job;
}
IngestJob getIngestJob() {
return job;
}
abstract void execute() throws InterruptedException;
}

View File

@ -0,0 +1,23 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
interface IngestTaskQueue {
IngestTask getNextTask() throws InterruptedException;
}

View File

@ -45,11 +45,11 @@ public final class RunIngestModulesDialog extends JDialog {
private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestDialog.title.text"); private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestDialog.title.text");
private static Dimension DIMENSIONS = new Dimension(500, 300); private static Dimension DIMENSIONS = new Dimension(500, 300);
private List<Content> dataSources = new ArrayList<>(); private List<Content> dataSources = new ArrayList<>();
private IngestJobLauncher ingestJobLauncher; private IngestJobConfigurator ingestJobLauncher;
public RunIngestModulesDialog(JFrame frame, String title, boolean modal) { public RunIngestModulesDialog(JFrame frame, String title, boolean modal) {
super(frame, title, modal); super(frame, title, modal);
ingestJobLauncher = new IngestJobLauncher(RunIngestModulesDialog.class.getCanonicalName()); ingestJobLauncher = new IngestJobConfigurator(RunIngestModulesDialog.class.getCanonicalName());
List<String> messages = ingestJobLauncher.getIngestJobConfigWarnings(); List<String> messages = ingestJobLauncher.getIngestJobConfigWarnings();
if (messages.isEmpty() == false) { if (messages.isEmpty() == false) {
StringBuilder warning = new StringBuilder(); StringBuilder warning = new StringBuilder();

View File

@ -32,7 +32,7 @@ import org.openide.util.NbBundle;
import org.openide.util.Utilities; import org.openide.util.Utilities;
import org.openide.util.Lookup; import org.openide.util.Lookup;
import org.openide.util.actions.Presenter; import org.openide.util.actions.Presenter;
import org.sleuthkit.autopsy.ingest.IngestJobLauncher; import org.sleuthkit.autopsy.ingest.IngestJobConfigurator;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskCoreException;

View File

@ -69,7 +69,7 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan
// Listen to the ingest modules to refresh the enabled/disabled state of // Listen to the ingest modules to refresh the enabled/disabled state of
// the components in sync with file ingest. // the components in sync with file ingest.
IngestManager.addPropertyChangeListener(new PropertyChangeListener() { IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
if (isIngestJobEvent(evt)) { if (isIngestJobEvent(evt)) {

View File

@ -126,7 +126,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
setButtonStates(); setButtonStates();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() { IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName(); String changed = evt.getPropertyName();

View File

@ -118,7 +118,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
ingestRunning = IngestManager.getInstance().isIngestRunning(); ingestRunning = IngestManager.getInstance().isIngestRunning();
updateComponents(); updateComponents();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() { IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName(); String changed = evt.getPropertyName();

0
c:casesSamll2Againautopsy.db Executable file
View File