mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-14 08:56:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #700 from rcordovano/ingest_improvements

Browse Source 
Ingest improvements
This commit is contained in:
Richard Cordovano 2014-05-02 15:19:55 -04:00
commit 15a05ddb90
29 changed files with 1282 additions and 1248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java
View File

@ -19,7 +19,7 @@
package org.sleuthkit.autopsy.casemodule; package org.sleuthkit.autopsy.casemodule;
import org.sleuthkit.autopsy.ingest.IngestJobLauncher; import org.sleuthkit.autopsy.ingest.IngestJobConfigurator;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import java.awt.Color; import java.awt.Color;
import java.awt.Component; import java.awt.Component;
@ -46,7 +46,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor;
class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel<WizardDescriptor> { class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel<WizardDescriptor> {
private static final Logger logger = Logger.getLogger(AddImageWizardIngestConfigPanel.class.getName()); private static final Logger logger = Logger.getLogger(AddImageWizardIngestConfigPanel.class.getName());
private IngestJobLauncher ingestConfig; private IngestJobConfigurator ingestConfig;
/** /**
* The visual component that displays this panel. If you need to access the * The visual component that displays this panel. If you need to access the
* component from this class, just use getComponent(). * component from this class, just use getComponent().
@ -73,7 +73,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel<WizardDe
this.progressPanel = proPanel; this.progressPanel = proPanel;
this.dataSourcePanel = dsPanel; this.dataSourcePanel = dsPanel;
ingestConfig = new IngestJobLauncher(AddImageWizardIngestConfigPanel.class.getCanonicalName()); ingestConfig = new IngestJobConfigurator(AddImageWizardIngestConfigPanel.class.getCanonicalName());
List<String> messages = ingestConfig.getIngestJobConfigWarnings(); List<String> messages = ingestConfig.getIngestJobConfigWarnings();
if (messages.isEmpty() == false) { if (messages.isEmpty() == false) {
StringBuilder warning = new StringBuilder(); StringBuilder warning = new StringBuilder();

11
Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.form
View File

@ -188,15 +188,12 @@
<Component class="javax.swing.JComboBox" name="numberOfFileIngestThreadsComboBox"> <Component class="javax.swing.JComboBox" name="numberOfFileIngestThreadsComboBox">
<Properties> <Properties>
<Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor"> <Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor">
<StringArray count="4"> <StringArray count="0"/>
<StringItem index="0" value="1"/>
<StringItem index="1" value="2"/>
<StringItem index="2" value="3"/>
<StringItem index="3" value="4"/>
</StringArray>
</Property> </Property>
<Property name="selectedIndex" type="int" value="1"/>
</Properties> </Properties>
<AuxValues>
<AuxValue name="JavaCodeGenerator_TypeParameters" type="java.lang.String" value="&lt;Integer&gt;"/>
</AuxValues>
</Component> </Component>
</SubComponents> </SubComponents>
</Form> </Form>

58
Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.corecomponents; package org.sleuthkit.autopsy.corecomponents;
import java.util.prefs.Preferences; import java.util.prefs.Preferences;
import javax.swing.DefaultComboBoxModel;
import org.openide.util.NbPreferences; import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManager;
@ -33,10 +34,35 @@ final class GeneralPanel extends javax.swing.JPanel {
GeneralPanel(GeneralOptionsPanelController controller) { GeneralPanel(GeneralOptionsPanelController controller) {
initComponents(); initComponents();
numberOfFileIngestThreadsComboBox.setModel(new DefaultComboBoxModel<>(new Integer[]{1, 2, 4, 8, 16}));
ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected());
// TODO listen to changes in form fields and call controller.changed() // TODO listen to changes in form fields and call controller.changed()
} }
void load() {
boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false);
keepCurrentViewerRB.setSelected(keepPreferredViewer);
useBestViewerRB.setSelected(!keepPreferredViewer);
boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true);
useLocalTimeRB.setSelected(useLocalTime);
useGMTTimeRB.setSelected(!useLocalTime);
dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false));
viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true));
numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getNumberOfFileIngestThreads());
}
void store() {
prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected());
prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected());
prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected());
prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected());
IngestManager.setNumberOfFileIngestThreads((Integer) numberOfFileIngestThreadsComboBox.getSelectedItem());
}
boolean valid() {
return true;
}
/** /**
* This method is called from within the constructor to initialize the form. * This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always * WARNING: Do NOT modify this code. The content of this method is always
@ -57,7 +83,7 @@ final class GeneralPanel extends javax.swing.JPanel {
dataSourcesHideKnownCB = new javax.swing.JCheckBox(); dataSourcesHideKnownCB = new javax.swing.JCheckBox();
viewsHideKnownCB = new javax.swing.JCheckBox(); viewsHideKnownCB = new javax.swing.JCheckBox();
jLabel4 = new javax.swing.JLabel(); jLabel4 = new javax.swing.JLabel();
numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox(); numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox<Integer>();
buttonGroup1.add(useBestViewerRB); buttonGroup1.add(useBestViewerRB);
useBestViewerRB.setSelected(true); useBestViewerRB.setSelected(true);
@ -97,9 +123,6 @@ final class GeneralPanel extends javax.swing.JPanel {
org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N
numberOfFileIngestThreadsComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "1", "2", "3", "4" }));
numberOfFileIngestThreadsComboBox.setSelectedIndex(1);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout); this.setLayout(layout);
layout.setHorizontalGroup( layout.setHorizontalGroup(
@ -165,31 +188,6 @@ final class GeneralPanel extends javax.swing.JPanel {
private void useGMTTimeRBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRBActionPerformed private void useGMTTimeRBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRBActionPerformed
ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected());
}//GEN-LAST:event_useGMTTimeRBActionPerformed }//GEN-LAST:event_useGMTTimeRBActionPerformed
void load() {
boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false);
keepCurrentViewerRB.setSelected(keepPreferredViewer);
useBestViewerRB.setSelected(!keepPreferredViewer);
boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true);
useLocalTimeRB.setSelected(useLocalTime);
useGMTTimeRB.setSelected(!useLocalTime);
dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false));
viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true));
numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getInstance().getNumberOfFileIngestThreads());
}
void store() {
prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected());
prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected());
prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected());
prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected());
IngestManager.getInstance().setNumberOfFileIngestThreads(Integer.valueOf(numberOfFileIngestThreadsComboBox.getSelectedItem().toString()));
}
boolean valid() {
// TODO check whether form is consistent and complete
return true;
}
// Variables declaration - do not modify//GEN-BEGIN:variables // Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.ButtonGroup buttonGroup1; private javax.swing.ButtonGroup buttonGroup1;
private javax.swing.ButtonGroup buttonGroup3; private javax.swing.ButtonGroup buttonGroup3;
@ -199,7 +197,7 @@ final class GeneralPanel extends javax.swing.JPanel {
private javax.swing.JLabel jLabel3; private javax.swing.JLabel jLabel3;
private javax.swing.JLabel jLabel4; private javax.swing.JLabel jLabel4;
private javax.swing.JRadioButton keepCurrentViewerRB; private javax.swing.JRadioButton keepCurrentViewerRB;
private javax.swing.JComboBox numberOfFileIngestThreadsComboBox; private javax.swing.JComboBox<Integer> numberOfFileIngestThreadsComboBox;
private javax.swing.JRadioButton useBestViewerRB; private javax.swing.JRadioButton useBestViewerRB;
private javax.swing.JRadioButton useGMTTimeRB; private javax.swing.JRadioButton useGMTTimeRB;
private javax.swing.JRadioButton useLocalTimeRB; private javax.swing.JRadioButton useLocalTimeRB;

3
Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java
View File

@ -131,7 +131,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
private void setListener() { private void setListener() {
Case.addPropertyChangeListener(this);// add this class to listen to any changes in the Case.java class Case.addPropertyChangeListener(this);// add this class to listen to any changes in the Case.java class
this.em.addPropertyChangeListener(this); this.em.addPropertyChangeListener(this);
IngestManager.addPropertyChangeListener(this); IngestManager.getInstance().addIngestJobEventListener(this);
IngestManager.getInstance().addIngestModuleEventListener(this);
} }
public void setDirectoryListingActive() { public void setDirectoryListingActive() {

10
Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties
View File

@ -52,12 +52,6 @@ Ensure the Case drive has at least 1GB free space and restart ingest.
IngestJobConfigurationPanel.advancedButton.text=Advanced IngestJobConfigurationPanel.advancedButton.text=Advanced
IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\: IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\:
IngestScheduler.FileSched.toString.curFiles.text=\
CurFiles, size\:
IngestScheduler.FileSched.toString.curDirs.text=\
CurDirs(stack), size\:
IngestScheduler.FileSched.toString.rootDirs.text=\
RootDirs(sorted), size\:
IngestManager.StartIngestJobsTask.run.displayName=Queueing ingest tasks IngestManager.StartIngestJobsTask.run.displayName=Queueing ingest tasks
IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...) IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...)
IngestManager.StartIngestJobsTask.run.catchException.msg=An error occurred while starting ingest. Results may only be partial IngestManager.StartIngestJobsTask.run.catchException.msg=An error occurred while starting ingest. Results may only be partial
@ -72,7 +66,3 @@ IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\:\
\ \
{0} {0}
IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=Ingest Failure IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=Ingest Failure
IngestManager.StartIngestJobsTask.run.progress.msg1=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg2=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg3=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg4=Data source ingest tasks for {0}

126
Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties
View File

@ -1,74 +1,64 @@
CTL_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8 CTL_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8
HINT_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8\u30A6\u30A3\u30F3\u30C9\u30A6 HINT_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8\u30a6\u30a3\u30f3\u30c9\u30a6
IngestDialog.closeButton.title=\u9589\u3058\u308B IngestDialog.closeButton.title=\u9589\u3058\u308b
IngestDialog.startButton.title=\u30B9\u30BF\u30FC\u30C8 IngestDialog.startButton.title=\u30b9\u30bf\u30fc\u30c8
IngestDialog.title.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB IngestDialog.title.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb
IngestJob.progress.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 IngestJob.progress.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09
IngestJob.progress.dataSourceIngest.displayName={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 IngestJob.progress.dataSourceIngest.displayName={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
IngestJob.progress.fileIngest.displayName={0}\u306E\u30D5\u30A1\u30A4\u30EB\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 IngestJob.progress.fileIngest.displayName={0}\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
IngestJobConfigurationPanel.advancedButton.actionCommand=\u30A2\u30C9\u30D0\u30F3\u30B9 IngestJobConfigurationPanel.advancedButton.actionCommand=\u30a2\u30c9\u30d0\u30f3\u30b9
IngestJobConfigurationPanel.advancedButton.text=\u30A2\u30C9\u30D0\u30F3\u30B9 IngestJobConfigurationPanel.advancedButton.text=\u30a2\u30c9\u30d0\u30f3\u30b9
IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u306E\u51E6\u7406 IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u306e\u51e6\u7406
IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524A\u9664\u3055\u308C\u305F\u30D5\u30A1\u30A4\u30EB\u7B49\u306E\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u51E6\u7406\u3002\u3088\u308A\u5B8C\u5168\u306A\u7D50\u679C\u304C\u51FA\u307E\u3059\u304C\u3001\u5927\u304D\u3044\u30A4\u30E1\u30FC\u30B8\u3067\u306F\u51E6\u7406\u6642\u9593\u304C\u9577\u304F\u306A\u308B\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524a\u9664\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb\u7b49\u306e\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u3092\u51e6\u7406\u3002\u3088\u308a\u5b8c\u5168\u306a\u7d50\u679c\u304c\u51fa\u307e\u3059\u304c\u3001\u5927\u304d\u3044\u30a4\u30e1\u30fc\u30b8\u3067\u306f\u51e6\u7406\u6642\u9593\u304c\u9577\u304f\u306a\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestManager.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC IngestManager.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc
IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestManager.StartIngestJobsTask.run.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 IngestManager.StartIngestJobsTask.run.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09
IngestManager.StartIngestJobsTask.run.catchException.msg=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u306E\u958B\u59CB\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002\u7D50\u679C\u304C\u4E00\u90E8\u306E\u3082\u306E IngestManager.StartIngestJobsTask.run.catchException.msg=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u306e\u958b\u59cb\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002\u7d50\u679c\u304c\u4e00\u90e8\u306e\u3082\u306e
IngestManager.StartIngestJobsTask.run.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF\u3092\u30AD\u30E5\u30FC\u30A4\u30F3\u30B0 IngestManager.StartIngestJobsTask.run.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af\u3092\u30ad\u30e5\u30fc\u30a4\u30f3\u30b0
IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30BD\u30FC\u30B9\u3001\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u3001\u8A73\u7D30\u304A\u3088\u3073\u30C7\u30FC\u30BF\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30bd\u30fc\u30b9\u3001\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u3001\u8a73\u7d30\u304a\u3088\u3073\u30c7\u30fc\u30bf\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.exception.srcSubjNotNull.msg=\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.srcSubjNotNull.msg=\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.exception.typeSrcSubjNotNull.msg=\u30E1\u30C3\u30BB\u30FC\u30B8\u30BF\u30A4\u30D7\u3001\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.typeSrcSubjNotNull.msg=\u30e1\u30c3\u30bb\u30fc\u30b8\u30bf\u30a4\u30d7\u3001\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.toString.data.text=\ \u30C7\u30FC\u30BF\uFF1A{0} IngestMessage.toString.data.text=\ \u30c7\u30fc\u30bf\uff1a{0}
IngestMessage.toString.date.text=\ \u65E5\u4ED8\uFF1A{0} IngestMessage.toString.date.text=\ \u65e5\u4ed8\uff1a{0}
IngestMessage.toString.details.text=\ \u8A73\u7D30\uFF1A{0} IngestMessage.toString.details.text=\ \u8a73\u7d30\uff1a{0}
IngestMessage.toString.subject.text=\ \u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\uFF1A{0} IngestMessage.toString.subject.text=\ \u30b5\u30d6\u30b8\u30a7\u30af\u30c8\uff1a{0}
IngestMessage.toString.type.text=\u30BF\u30A4\u30D7\uFF1A{0} IngestMessage.toString.type.text=\u30bf\u30a4\u30d7\uff1a{0}
IngestMessageDetailsPanel.copyMenuItem.text=\u30B3\u30D4\u30FC IngestMessageDetailsPanel.copyMenuItem.text=\u30b3\u30d4\u30fc
IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30C6\u30AD\u30B9\u30C8\uFF0Fhtml IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30c6\u30ad\u30b9\u30c8\uff0fhtml
IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629E IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629e
IngestMessageDetailsPanel.viewArtifactButton.text=\u7D50\u679C\u3078\u79FB\u52D5 IngestMessageDetailsPanel.viewArtifactButton.text=\u7d50\u679c\u3078\u79fb\u52d5
IngestMessageDetailsPanel.viewContentButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u3078\u79FB\u52D5 IngestMessageDetailsPanel.viewContentButton.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3078\u79fb\u52d5
IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30D6\u30FC\u30EB\u5024\u3067\u306F\u306A\u3044\u3082\u306E\u306BBooleanRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30d6\u30fc\u30eb\u5024\u3067\u306f\u306a\u3044\u3082\u306e\u306bBooleanRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f
IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65E5\u4ED8\u3067\u306F\u306A\u3044\u3082\u306E\u306BDateRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F\u3002 IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65e5\u4ed8\u3067\u306f\u306a\u3044\u3082\u306e\u306bDateRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f\u3002
IngestMessagePanel.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC IngestMessagePanel.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc
IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestMessagePanel.MsgTableMod.colNames.module=\u30E2\u30B8\u30E5\u30FC\u30EB IngestMessagePanel.MsgTableMod.colNames.module=\u30e2\u30b8\u30e5\u30fc\u30eb
IngestMessagePanel.MsgTableMod.colNames.new=\u65B0\u898F\uFF1F IngestMessagePanel.MsgTableMod.colNames.new=\u65b0\u898f\uff1f
IngestMessagePanel.MsgTableMod.colNames.num=\u756A\u53F7 IngestMessagePanel.MsgTableMod.colNames.num=\u756a\u53f7
IngestMessagePanel.MsgTableMod.colNames.subject=\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8 IngestMessagePanel.MsgTableMod.colNames.subject=\u30b5\u30d6\u30b8\u30a7\u30af\u30c8
IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30BF\u30A4\u30E0\u30B9\u30BF\u30F3\u30D7 IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30bf\u30a4\u30e0\u30b9\u30bf\u30f3\u30d7
IngestMessagePanel.sortByComboBox.model.priority=\u512A\u5148\u5EA6 IngestMessagePanel.sortByComboBox.model.priority=\u512a\u5148\u5ea6
IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593 IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593
IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uFF08\u6642\u7CFB\u5217\uFF09\u307E\u305F\u306F\u30E1\u30C3\u30BB\u30FC\u30B8\u306E\u512A\u5148\u5EA6\u3067\u30BD\u30FC\u30C8 IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uff08\u6642\u7cfb\u5217\uff09\u307e\u305f\u306f\u30e1\u30c3\u30bb\u30fc\u30b8\u306e\u512a\u5148\u5ea6\u3067\u30bd\u30fc\u30c8
IngestMessagePanel.sortByLabel.text=\u4E0B\u8A18\u3067\u30BD\u30FC\u30C8\uFF1A IngestMessagePanel.sortByLabel.text=\u4e0b\u8a18\u3067\u30bd\u30fc\u30c8\uff1a
IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8A08\uFF1A IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8a08\uff1a
IngestMessagePanel.totalMessagesNameVal.text=- IngestMessagePanel.totalMessagesNameVal.text=-
IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30E6\u30CB\u30FC\u30AF\uFF1A IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30e6\u30cb\u30fc\u30af\uff1a
IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestMessagePanel.totalUniqueMessagesNameVal.text=-
IngestMessagesToolbar.customizeButton.toolTipText=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E1\u30C3\u30BB\u30FC\u30B8 IngestMessagesToolbar.customizeButton.toolTipText=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e1\u30c3\u30bb\u30fc\u30b8
IngestMessageTopComponent.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 IngestMessageTopComponent.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9
IngestMessageTopComponent.displayReport.option.GenRpt=\u30EC\u30DD\u30FC\u30C8\u751F\u6210 IngestMessageTopComponent.displayReport.option.GenRpt=\u30ec\u30dd\u30fc\u30c8\u751f\u6210
IngestMessageTopComponent.displayReport.option.OK=OK IngestMessageTopComponent.displayReport.option.OK=OK
IngestMessageTopComponent.initComponents.name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 IngestMessageTopComponent.initComponents.name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9
IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30EC\u30DD\u30FC\u30C8 IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30ec\u30dd\u30fc\u30c8
IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30C7\u30A3\u30B9\u30AF{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3\u306E\u305F\u3081\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u4E2D\u6B62\u3057\u307E\u3059\u3002\ IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3\u306e\u305f\u3081\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u4e2d\u6b62\u3057\u307e\u3059\u3002\
\u30B1\u30FC\u30B9\u30C9\u30E9\u30A4\u30D6\u306B\u6700\u4F4E1GB\u306E\u7A7A\u304D\u9818\u57DF\u304C\u3042\u308B\u306E\u3092\u78BA\u8A8D\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u518D\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 \u30b1\u30fc\u30b9\u30c9\u30e9\u30a4\u30d6\u306b\u6700\u4f4e1GB\u306e\u7a7a\u304d\u9818\u57df\u304c\u3042\u308b\u306e\u3092\u78ba\u8a8d\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u518d\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002
IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u304C\u4E2D\u6B62\u3055\u308C\u307E\u3057\u305F\u30FC{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3 IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30B5\u30A4\u30BA\uFF1A IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30b5\u30a4\u30ba\uff1a
IngestScheduler.FileSched.toString.curDirs.text=\ OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
CurDirs(stack), \u30B5\u30A4\u30BA\uFF1A IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30a8\u30e9\u30fc\uff1a\
IngestScheduler.FileSched.toString.curFiles.text=\
CurFiles, \u30B5\u30A4\u30BA\uFF1A
IngestScheduler.FileSched.toString.rootDirs.text=\
RootDirs(sorted), \u30B5\u30A4\u30BA\uFF1A
OpenIDE-Module-Name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8
IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30A8\u30E9\u30FC\uFF1A\
\ \
{0} {0}
IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uFF11\u3064\u307E\u305F\u306F\u8907\u6570\u306E\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u30B9\u30BF\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u306F\u30AD\u30E3\u30F3\u30BB\u30EB\u3055\u308C\u307E\u3057\u305F\u3002 IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uff11\u3064\u307e\u305f\u306f\u8907\u6570\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u30b9\u30bf\u30fc\u30c8\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\u3002\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u306f\u30ad\u30e3\u30f3\u30bb\u30eb\u3055\u308c\u307e\u3057\u305f\u3002
IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305F\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u7121\u52B9\u5316\u3059\u308B\u304B\u30A8\u30E9\u30FC\u3092\u89E3\u6C7A\u3057\u3001\u305D\u306E\u5F8C\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u53F3\u30AF\u30EA\u30C3\u30AF\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u5B9F\u884C\u3092\u9078\u629E\u3057\u3066\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u30EA\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305f\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u7121\u52b9\u5316\u3059\u308b\u304b\u30a8\u30e9\u30fc\u3092\u89e3\u6c7a\u3057\u3001\u305d\u306e\u5f8c\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u53f3\u30af\u30ea\u30c3\u30af\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u5b9f\u884c\u3092\u9078\u629e\u3057\u3066\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u30ea\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002
IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u5931\u6557 IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u5931\u6557

14
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java
View File

@ -18,16 +18,18 @@
*/ */
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
import org.netbeans.api.progress.ProgressHandle;
/** /**
* Used by data source ingest modules to report progress. * Used by data source ingest modules to report progress.
*/ */
public class DataSourceIngestModuleProgress { public class DataSourceIngestModuleProgress {
private final IngestJob ingestJob; private final ProgressHandle progress;
private final String moduleDisplayName; private final String moduleDisplayName;
DataSourceIngestModuleProgress(IngestJob ingestJob, String moduleDisplayName) { DataSourceIngestModuleProgress(ProgressHandle progress, String moduleDisplayName) {
this.ingestJob = ingestJob; this.progress = progress;
this.moduleDisplayName = moduleDisplayName; this.moduleDisplayName = moduleDisplayName;
} }
@ -40,7 +42,7 @@ public class DataSourceIngestModuleProgress {
* data source. * data source.
*/ */
public void switchToDeterminate(int workUnits) { public void switchToDeterminate(int workUnits) {
ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits); progress.switchToDeterminate(workUnits);
} }
/** /**
@ -48,7 +50,7 @@ public class DataSourceIngestModuleProgress {
* the total work units to process the data source is unknown. * the total work units to process the data source is unknown.
*/ */
public void switchToIndeterminate() { public void switchToIndeterminate() {
ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate(); progress.switchToIndeterminate();
} }
/** /**
@ -58,6 +60,6 @@ public class DataSourceIngestModuleProgress {
* @param workUnits Number of work units performed so far by the module. * @param workUnits Number of work units performed so far by the module.
*/ */
public void progress(int workUnits) { public void progress(int workUnits) {
ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits); progress.progress(this.moduleDisplayName, workUnits);
} }
} }

18
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java
View File

@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.netbeans.api.progress.ProgressHandle;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
/** /**
@ -30,12 +31,12 @@ import org.sleuthkit.datamodel.Content;
*/ */
final class DataSourceIngestPipeline { final class DataSourceIngestPipeline {
private final IngestJob job; private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>(); private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
DataSourceIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) { DataSourceIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.job = task; this.context = context;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
} }
@ -50,7 +51,6 @@ final class DataSourceIngestPipeline {
for (IngestModuleTemplate template : moduleTemplates) { for (IngestModuleTemplate template : moduleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) { if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try { try {
module.startUp(context); module.startUp(context);
modulesByClass.put(module.getClassName(), module); modulesByClass.put(module.getClassName(), module);
@ -75,26 +75,26 @@ final class DataSourceIngestPipeline {
return errors; return errors;
} }
List<IngestModuleError> process() { List<IngestModuleError> process(Content dataSource, ProgressHandle progress) {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) { for (DataSourceIngestModuleDecorator module : this.modules) {
try { try {
module.process(job.getDataSource(), new DataSourceIngestModuleProgress(job, module.getDisplayName())); module.process(dataSource, new DataSourceIngestModuleProgress(progress, module.getDisplayName()));
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }
if (job.isCancelled()) { if (context.isJobCancelled()) {
break; break;
} }
} }
return errors; return errors;
} }
List<IngestModuleError> shutDown(boolean ingestJobCancelled) { List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) { for (DataSourceIngestModuleDecorator module : this.modules) {
try { try {
module.shutDown(ingestJobCancelled); module.shutDown(context.isJobCancelled());
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }

40
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java Executable file
View File

@ -0,0 +1,40 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import org.sleuthkit.datamodel.Content;
final class DataSourceIngestTask extends IngestTask {
private final Content dataSource;
DataSourceIngestTask(IngestJob ingestJob, Content dataSource) {
super(ingestJob);
this.dataSource = dataSource;
}
Content getDataSource() {
return dataSource;
}
@Override
void execute() throws InterruptedException {
getIngestJob().process(dataSource);
}
}

17
Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java
View File

@ -30,12 +30,12 @@ import org.sleuthkit.datamodel.AbstractFile;
*/ */
final class FileIngestPipeline { final class FileIngestPipeline {
private final IngestJob job; private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private List<FileIngestModuleDecorator> modules = new ArrayList<>(); private List<FileIngestModuleDecorator> modules = new ArrayList<>();
FileIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) { FileIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.job = task; this.context = context;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
} }
@ -50,7 +50,6 @@ final class FileIngestPipeline {
for (IngestModuleTemplate template : moduleTemplates) { for (IngestModuleTemplate template : moduleTemplates) {
if (template.isFileIngestModuleTemplate()) { if (template.isFileIngestModuleTemplate()) {
FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try { try {
module.startUp(context); module.startUp(context);
modulesByClass.put(module.getClassName(), module); modulesByClass.put(module.getClassName(), module);
@ -83,22 +82,22 @@ final class FileIngestPipeline {
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }
if (job.isCancelled()) { if (context.isJobCancelled()) {
break; break;
} }
} }
file.close(); file.close();
if (!job.isCancelled()) { if (!context.isJobCancelled()) {
IngestManager.fireFileIngestDone(file.getId()); IngestManager.getInstance().fireFileIngestDone(file.getId());
} }
return errors; return errors;
} }
List<IngestModuleError> shutDown(boolean ingestJobCancelled) { List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) { for (FileIngestModuleDecorator module : this.modules) {
try { try {
module.shutDown(ingestJobCancelled); module.shutDown(context.isJobCancelled());
} catch (Exception ex) { } catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex)); errors.add(new IngestModuleError(module.getDisplayName(), ex));
} }

69
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java Executable file
View File

@ -0,0 +1,69 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.Objects;
import org.sleuthkit.datamodel.AbstractFile;
final class FileIngestTask extends IngestTask {
private final AbstractFile file;
FileIngestTask(IngestJob job, AbstractFile file) {
super(job);
this.file = file;
}
AbstractFile getFile() {
return file;
}
@Override
void execute() throws InterruptedException {
getIngestJob().process(file);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
FileIngestTask other = (FileIngestTask) obj;
IngestJob job = getIngestJob();
IngestJob otherJob = other.getIngestJob();
if (job != otherJob && (job == null || !job.equals(otherJob))) {
return false;
}
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
return false;
}
return true;
}
@Override
public int hashCode() {
int hash = 5;
hash = 47 * hash + Objects.hashCode(getIngestJob());
hash = 47 * hash + Objects.hashCode(this.file);
return hash;
}
}

100
Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java Executable file
View File

@ -0,0 +1,100 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
* Get counts of ingestable files/dirs for the content input source.
*
* Note, also includes counts of all unalloc children files (for the fs, image,
* volume) even if ingest didn't ask for them
*/
final class GetFilesCountVisitor extends ContentVisitor.Default<Long> {
private static final Logger logger = Logger.getLogger(GetFilesCountVisitor.class.getName());
@Override
public Long visit(FileSystem fs) {
//recursion stop here
//case of a real fs, query all files for it
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
StringBuilder queryB = new StringBuilder();
queryB.append("( (fs_obj_id = ").append(fs.getId()); //NON-NLS
//queryB.append(") OR (fs_obj_id = NULL) )");
queryB.append(") )");
queryB.append(" AND ( (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS
queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS
queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_VIRT.getValue()); //NON-NLS
queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS
queryB.append(") )");
//queryB.append( "AND (type = ");
//queryB.append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.getFileType());
//queryB.append(")");
try {
final String query = queryB.toString();
logger.log(Level.INFO, "Executing count files query: {0}", query); //NON-NLS
return sc.countFilesWhere(query);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); //NON-NLS
return 0L;
}
}
@Override
public Long visit(LayoutFile lf) {
//recursion stop here
//case of LayoutFile child of Image or Volume
return 1L;
}
private long getCountFromChildren(Content content) {
long count = 0;
try {
List<Content> children = content.getChildren();
if (children.size() > 0) {
for (Content child : children) {
count += child.accept(this);
}
} else {
count = 1;
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not get count of objects from children to get num of total files to be ingested", ex); //NON-NLS
}
return count;
}
@Override
protected Long defaultVisit(Content cntnt) {
//recurse assuming this is image/vs/volume
//recursion stops at fs or unalloc file
return getCountFromChildren(cntnt);
}
}

88
Core/src/org/sleuthkit/autopsy/ingest/GetRootDirectoryVisitor.java Executable file
View File

@ -0,0 +1,88 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.VirtualDirectory;
/**
* Finds top level objects such as file system root directories, layout
* files and virtual directories.
*/
final class GetRootDirectoryVisitor extends GetFilesContentVisitor {
@Override
public Collection<AbstractFile> visit(VirtualDirectory ld) {
//case when we hit a layout directoryor local file container, not under a real FS
//or when root virt dir is scheduled
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(ld);
return ret;
}
@Override
public Collection<AbstractFile> visit(LayoutFile lf) {
//case when we hit a layout file, not under a real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(lf);
return ret;
}
@Override
public Collection<AbstractFile> visit(Directory drctr) {
//we hit a real directory, a child of real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(drctr);
return ret;
}
@Override
public Collection<AbstractFile> visit(FileSystem fs) {
return getAllFromChildren(fs);
}
@Override
public Collection<AbstractFile> visit(File file) {
//can have derived files
return getAllFromChildren(file);
}
@Override
public Collection<AbstractFile> visit(DerivedFile derivedFile) {
//can have derived files
//TODO test this and overall scheduler with derived files
return getAllFromChildren(derivedFile);
}
@Override
public Collection<AbstractFile> visit(LocalFile localFile) {
//can have local files
//TODO test this and overall scheduler with local files
return getAllFromChildren(localFile);
}
}

254
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -19,83 +19,111 @@
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory; import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable; import org.openide.util.Cancellable;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
/**
* Encapsulates a data source and the ingest module pipelines to be used to
* ingest the data source.
*/
final class IngestJob { final class IngestJob {
private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private final long id; private final long id;
private final Content dataSource; private final Content rootDataSource;
private final List<IngestModuleTemplate> ingestModuleTemplates; private final List<IngestModuleTemplate> ingestModuleTemplates;
private final boolean processUnallocatedSpace; private final boolean processUnallocatedSpace;
private final HashMap<Long, FileIngestPipeline> fileIngestPipelines = new HashMap<>(); private final LinkedBlockingQueue<DataSourceIngestPipeline> dataSourceIngestPipelines = new LinkedBlockingQueue<>();
private final HashMap<Long, DataSourceIngestPipeline> dataSourceIngestPipelines = new HashMap<>(); private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelines = new LinkedBlockingQueue<>();
private final IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); private long estimatedFilesToProcess = 0L; // Guarded by this
private FileIngestPipeline initialFileIngestPipeline = null; private long processedFiles = 0L; // Guarded by this
private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; private ProgressHandle dataSourceTasksProgress;
private ProgressHandle dataSourceTaskProgress;
private ProgressHandle fileTasksProgress; private ProgressHandle fileTasksProgress;
int totalEnqueuedFiles = 0; private volatile boolean cancelled = false;
private int processedFiles = 0;
private volatile boolean cancelled;
IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) { IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) {
this.id = id; this.id = id;
this.dataSource = dataSource; this.rootDataSource = dataSource;
this.ingestModuleTemplates = ingestModuleTemplates; this.ingestModuleTemplates = ingestModuleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace; this.processUnallocatedSpace = processUnallocatedSpace;
this.cancelled = false;
} }
long getId() { long getId() {
return id; return id;
} }
Content getDataSource() {
return dataSource;
}
boolean shouldProcessUnallocatedSpace() { boolean shouldProcessUnallocatedSpace() {
return processUnallocatedSpace; return processUnallocatedSpace;
} }
synchronized List<IngestModuleError> startUpIngestPipelines() { List<IngestModuleError> startUp() throws InterruptedException {
startDataSourceIngestProgressBar(); List<IngestModuleError> errors = startUpIngestPipelines();
if (errors.isEmpty()) {
startFileIngestProgressBar(); startFileIngestProgressBar();
return startUpInitialIngestPipelines(); startDataSourceIngestProgressBar();
}
return errors;
}
private List<IngestModuleError> startUpIngestPipelines() throws InterruptedException {
IngestJobContext context = new IngestJobContext(this);
List<IngestModuleError> errors = new ArrayList<>();
int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
dataSourceIngestPipelines.put(pipeline);
if (!errors.isEmpty()) {
// No need to accumulate presumably redundant errors.
break;
}
}
maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
FileIngestPipeline pipeline = new FileIngestPipeline(context, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
fileIngestPipelines.put(pipeline);
if (!errors.isEmpty()) {
// No need to accumulate presumably redundant errors.
break;
}
}
logIngestModuleErrors(errors);
return errors; // Returned so UI can report to user.
} }
private void startDataSourceIngestProgressBar() { private void startDataSourceIngestProgressBar() {
final String displayName = NbBundle final String displayName = NbBundle.getMessage(this.getClass(),
.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", this.dataSource.getName()); "IngestJob.progress.dataSourceIngest.displayName",
dataSourceTaskProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { rootDataSource.getName());
dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
if (dataSourceTaskProgress != null) { if (dataSourceTasksProgress != null) {
dataSourceTaskProgress.setDisplayName(NbBundle.getMessage(this.getClass(), dataSourceTasksProgress.setDisplayName(
NbBundle.getMessage(this.getClass(),
"IngestJob.progress.cancelling", "IngestJob.progress.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); IngestJob.this.cancel();
return true; return true;
} }
}); });
dataSourceTaskProgress.start(); dataSourceTasksProgress.start();
dataSourceTaskProgress.switchToIndeterminate(); dataSourceTasksProgress.switchToIndeterminate();
} }
private void startFileIngestProgressBar() { private void startFileIngestProgressBar() {
final String displayName = NbBundle final String displayName = NbBundle.getMessage(this.getClass(),
.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); "IngestJob.progress.fileIngest.displayName",
rootDataSource.getName());
fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
@ -104,124 +132,84 @@ final class IngestJob {
NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); IngestJob.this.cancel();
return true; return true;
} }
}); });
estimatedFilesToProcess = rootDataSource.accept(new GetFilesCountVisitor());
fileTasksProgress.start(); fileTasksProgress.start();
fileTasksProgress.switchToIndeterminate(); fileTasksProgress.switchToDeterminate((int) estimatedFilesToProcess);
totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
} }
private List<IngestModuleError> startUpInitialIngestPipelines() { void process(Content dataSource) throws InterruptedException {
// Create a per thread instance of each pipeline type right now to make // If the job is not cancelled, complete the task, otherwise just flush
// (reasonably) sure that the ingest modules can be started. // it. In either case, the task counter needs to be decremented and the
initialDataSourceIngestPipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); // shut down check needs to occur.
initialFileIngestPipeline = new FileIngestPipeline(this, ingestModuleTemplates); if (!isCancelled()) {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(initialDataSourceIngestPipeline.startUp()); DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take();
errors.addAll(initialFileIngestPipeline.startUp()); errors.addAll(pipeline.process(dataSource, dataSourceTasksProgress));
return errors; if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
dataSourceIngestPipelines.put(pipeline);
}
} }
synchronized DataSourceIngestPipeline getDataSourceIngestPipelineForThread(long threadId) { void process(AbstractFile file) throws InterruptedException {
DataSourceIngestPipeline pipeline; // If the job is not cancelled, complete the task, otherwise just flush
if (initialDataSourceIngestPipeline != null) { // it. In either case, the task counter needs to be decremented and the
pipeline = initialDataSourceIngestPipeline; // shut down check needs to occur.
initialDataSourceIngestPipeline = null; if (!isCancelled()) {
dataSourceIngestPipelines.put(threadId, pipeline);
} else if (!dataSourceIngestPipelines.containsKey(threadId)) {
pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates);
pipeline.startUp();
dataSourceIngestPipelines.put(threadId, pipeline);
} else {
pipeline = dataSourceIngestPipelines.get(threadId);
}
return pipeline;
}
synchronized FileIngestPipeline getFileIngestPipelineForThread(long threadId) {
FileIngestPipeline pipeline;
if (initialFileIngestPipeline != null) {
pipeline = initialFileIngestPipeline;
initialFileIngestPipeline = null;
fileIngestPipelines.put(threadId, pipeline);
} else if (!fileIngestPipelines.containsKey(threadId)) {
pipeline = new FileIngestPipeline(this, ingestModuleTemplates);
pipeline.startUp();
fileIngestPipelines.put(threadId, pipeline);
} else {
pipeline = fileIngestPipelines.get(threadId);
}
return pipeline;
}
synchronized List<IngestModuleError> releaseIngestPipelinesForThread(long threadId) {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
synchronized (this) {
DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId);
if (dataSourceIngestPipeline != null) {
errors.addAll(dataSourceIngestPipeline.shutDown(cancelled));
dataSourceIngestPipelines.remove(threadId);
}
if (initialDataSourceIngestPipeline == null && dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) {
dataSourceTaskProgress.finish();
dataSourceTaskProgress = null;
}
FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId);
if (fileIngestPipeline != null) {
errors.addAll(fileIngestPipeline.shutDown(cancelled));
fileIngestPipelines.remove(threadId);
}
if (initialFileIngestPipeline == null && fileIngestPipelines.isEmpty() && fileTasksProgress != null) {
fileTasksProgress.finish();
fileTasksProgress = null;
}
return errors;
}
synchronized boolean areIngestPipelinesShutDown() {
return (initialDataSourceIngestPipeline == null
&& dataSourceIngestPipelines.isEmpty()
&& initialFileIngestPipeline == null
&& fileIngestPipelines.isEmpty());
}
synchronized ProgressHandle getDataSourceTaskProgressBar() {
return this.dataSourceTaskProgress;
}
synchronized void updateFileTasksProgressBar(String currentFileName) {
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
totalEnqueuedFiles = newTotalEnqueuedFiles + 1;
fileTasksProgress.switchToIndeterminate();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
}
if (processedFiles < totalEnqueuedFiles) {
++processedFiles; ++processedFiles;
if (processedFiles <= estimatedFilesToProcess) {
fileTasksProgress.progress(file.getName(), (int) processedFiles);
} else {
fileTasksProgress.progress(file.getName(), (int) estimatedFilesToProcess);
}
}
FileIngestPipeline pipeline = fileIngestPipelines.take();
errors.addAll(pipeline.process(file));
fileIngestPipelines.put(pipeline);
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
}
} }
fileTasksProgress.progress(currentFileName, processedFiles); void shutDown() {
List<IngestModuleError> errors = new ArrayList<>();
while (!dataSourceIngestPipelines.isEmpty()) {
DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll();
errors.addAll(pipeline.shutDown());
}
while (!fileIngestPipelines.isEmpty()) {
FileIngestPipeline pipeline = fileIngestPipelines.poll();
errors.addAll(pipeline.shutDown());
}
fileTasksProgress.finish();
dataSourceTasksProgress.finish();
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
} }
synchronized void cancel() { private void logIngestModuleErrors(List<IngestModuleError> errors) {
if (initialDataSourceIngestPipeline != null) { for (IngestModuleError error : errors) {
initialDataSourceIngestPipeline.shutDown(true); logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError());
initialDataSourceIngestPipeline = null;
} }
if (initialFileIngestPipeline != null) {
initialFileIngestPipeline.shutDown(true);
initialFileIngestPipeline = null;
}
cancelled = true;
} }
boolean isCancelled() { boolean isCancelled() {
return cancelled; return cancelled;
} }
void cancel() {
cancelled = true;
fileTasksProgress.finish();
dataSourceTasksProgress.finish();
IngestManager.getInstance().fireIngestJobCancelled(id);
}
} }

6
Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java → Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java
View File

@ -41,14 +41,14 @@ import org.sleuthkit.datamodel.Content;
* for a particular context and for launching ingest jobs that process one or * for a particular context and for launching ingest jobs that process one or
* more data sources using the ingest job configuration. * more data sources using the ingest job configuration.
*/ */
public final class IngestJobLauncher { public final class IngestJobConfigurator {
private static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; //NON-NLS private static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; //NON-NLS
private static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; //NON-NLS private static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; //NON-NLS
private static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; //NON-NLS private static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; //NON-NLS
private static final String MODULE_SETTINGS_FOLDER_PATH = new StringBuilder(PlatformUtil.getUserConfigDirectory()).append(File.separator).append("IngestModuleSettings").toString(); //NON-NLS private static final String MODULE_SETTINGS_FOLDER_PATH = new StringBuilder(PlatformUtil.getUserConfigDirectory()).append(File.separator).append("IngestModuleSettings").toString(); //NON-NLS
private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS
private static final Logger logger = Logger.getLogger(IngestJobLauncher.class.getName()); private static final Logger logger = Logger.getLogger(IngestJobConfigurator.class.getName());
private final String launcherContext; private final String launcherContext;
private String moduleSettingsFolderForContext = null; private String moduleSettingsFolderForContext = null;
private final List<String> warnings = new ArrayList<>(); private final List<String> warnings = new ArrayList<>();
@ -61,7 +61,7 @@ public final class IngestJobLauncher {
* *
* @param launcherContext The context identifier. * @param launcherContext The context identifier.
*/ */
public IngestJobLauncher(String launcherContext) { public IngestJobConfigurator(String launcherContext) {
this.launcherContext = launcherContext; this.launcherContext = launcherContext;
createModuleSettingsFolderForContext(); createModuleSettingsFolderForContext();

2
Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java
View File

@ -60,7 +60,7 @@ public final class IngestJobContext {
*/ */
public void addFiles(List<AbstractFile> files) { public void addFiles(List<AbstractFile> files) {
for (AbstractFile file : files) { for (AbstractFile file : files) {
IngestManager.getInstance().addFileToIngestJob(ingestJob.getId(), file); IngestScheduler.getInstance().addFileToIngestJob(ingestJob, file);
} }
} }
} }

587
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -20,10 +20,10 @@ package org.sleuthkit.autopsy.ingest;
import java.beans.PropertyChangeListener; import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport; import java.beans.PropertyChangeSupport;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.CancellationException; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
@ -36,72 +36,148 @@ import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable; import org.openide.util.Cancellable;
import org.openide.util.NbPreferences; import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
import java.util.prefs.Preferences; import java.util.prefs.Preferences;
import javax.swing.JOptionPane; import javax.swing.JOptionPane;
import javax.swing.SwingWorker;
import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask;
/** /**
* Manages the execution of ingest jobs. * Manages the execution of ingest jobs.
*/ */
public class IngestManager { public class IngestManager {
private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1;
private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS
private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1;
private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16;
private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2;
private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class);
private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class);
private static final IngestManager instance = new IngestManager(); private static final IngestManager instance = new IngestManager();
private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final IngestScheduler scheduler = IngestScheduler.getInstance(); private final IngestScheduler scheduler = IngestScheduler.getInstance();
private final IngestMonitor ingestMonitor = new IngestMonitor(); private final IngestMonitor ingestMonitor = new IngestMonitor();
private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS);
private final ExecutorService fireEventTasksExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor();
private final ConcurrentHashMap<Long, IngestJob> ingestJobs = new ConcurrentHashMap<>(1, 0.9f, 4); // Maps job ids to jobs. private final AtomicLong nextThreadId = new AtomicLong(0L);
private final ConcurrentHashMap<Long, Future<?>> ingestTasks = new ConcurrentHashMap<>(); // Maps task ids to task cancellation handles. Guarded by this. private final ConcurrentHashMap<Long, Future<Void>> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final AtomicLong ingestJobId = new AtomicLong(0L); private final ConcurrentHashMap<Long, Future<?>> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final AtomicLong ingestTaskId = new AtomicLong(0L); private final ConcurrentHashMap<Long, Future<?>> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private volatile IngestMessageTopComponent ingestMessageBox; private volatile IngestMessageTopComponent ingestMessageBox;
/** /**
* Gets the IngestManager singleton, creating it if necessary. * Gets the ingest manager.
* *
* @returns The IngestManager singleton. * @returns A singleton IngestManager object.
*/ */
public static IngestManager getInstance() { public static IngestManager getInstance() {
return instance; return instance;
} }
/**
* Starts the ingest monitor and the data source ingest and file ingest
* threads.
*/
private IngestManager() { private IngestManager() {
startDataSourceIngestThread();
int numberOfFileIngestThreads = getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfFileIngestThreads; ++i) {
startFileIngestThread();
}
} }
/** /**
* Signals to the ingest manager that it can go find the top component for * Signals to the ingest manager that it can go about finding the top
* the ingest messages in box. Called by the custom installer for this * component for the ingest messages in box. Called by the custom installer
* package once the window system is initialized. * for this package once the window system is initialized.
*/ */
void initIngestMessageInbox() { void initIngestMessageInbox() {
if (this.ingestMessageBox == null) { if (ingestMessageBox == null) {
this.ingestMessageBox = IngestMessageTopComponent.findInstance(); ingestMessageBox = IngestMessageTopComponent.findInstance();
} }
} }
/**
* Gets the maximum number of data source ingest threads the ingest manager
* will use.
*/
public static int getMaxNumberOfDataSourceIngestThreads() {
return MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS;
}
/**
* Gets the maximum number of file ingest threads the ingest manager will
* use.
*/
public static int getMaxNumberOfFileIngestThreads() {
return MAX_NUMBER_OF_FILE_INGEST_THREADS;
}
/**
* Gets the number of file ingest threads the ingest manager will use.
*/
public synchronized static int getNumberOfFileIngestThreads() { public synchronized static int getNumberOfFileIngestThreads() {
return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS);
} }
/**
* Changes the number of file ingest threads the ingest manager will use to
* no more than MAX_NUMBER_OF_FILE_INGEST_THREADS and no less than
* MIN_NUMBER_OF_FILE_INGEST_THREADS. Out of range requests are converted to
* requests for DEFAULT_NUMBER_OF_FILE_INGEST_THREADS.
*
* @param numberOfThreads The desired number of file ingest threads.
*/
public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) { public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) {
if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS if ((numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) {
|| numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) {
numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS;
} }
userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads);
if (instance.fileIngestThreads.size() != numberOfThreads) {
if (instance.fileIngestThreads.size() > numberOfThreads) {
Long[] threadIds = instance.fileIngestThreads.keySet().toArray(new Long[instance.fileIngestThreads.size()]);
int numberOfThreadsToCancel = instance.fileIngestThreads.size() - numberOfThreads;
for (int i = 0; i < numberOfThreadsToCancel; ++i) {
instance.cancelFileIngestThread(threadIds[i]);
}
} else if (instance.fileIngestThreads.size() < numberOfThreads) {
int numberOfThreadsToAdd = numberOfThreads - instance.fileIngestThreads.size();
for (int i = 0; i < numberOfThreadsToAdd; ++i) {
instance.startFileIngestThread();
}
}
}
}
/**
* Submits a DataSourceIngestThread Runnable to the data source ingest
* thread pool.
*/
private void startDataSourceIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = dataSourceIngestThreadPool.submit(new ExecuteIngestTasksThread(scheduler.getDataSourceIngestTaskQueue()));
dataSourceIngestThreads.put(threadId, handle);
}
/**
* Submits a DataSourceIngestThread Runnable to the data source ingest
* thread pool.
*/
private void startFileIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = fileIngestThreadPool.submit(new ExecuteIngestTasksThread(scheduler.getFileIngestTaskQueue()));
fileIngestThreads.put(threadId, handle);
}
/**
* Cancels a DataSourceIngestThread Runnable in the file ingest thread pool.
*/
private void cancelFileIngestThread(long threadId) {
Future<?> handle = fileIngestThreads.remove(threadId);
handle.cancel(true);
} }
synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) { synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
@ -109,9 +185,9 @@ public class IngestManager {
ingestMessageBox.clearMessages(); ingestMessageBox.clearMessages();
} }
long taskId = ingestTaskId.incrementAndGet(); long taskId = nextThreadId.incrementAndGet();
Future<?> task = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); Future<Void> task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
ingestTasks.put(taskId, task); startIngestJobThreads.put(taskId, task);
if (ingestMessageBox != null) { if (ingestMessageBox != null) {
ingestMessageBox.restoreMessages(); ingestMessageBox.restoreMessages();
@ -121,21 +197,31 @@ public class IngestManager {
/** /**
* Test if any ingest jobs are in progress. * Test if any ingest jobs are in progress.
* *
* @return True if any ingest jobs are in progress, false otherwise * @return True if any ingest jobs are in progress, false otherwise.
*/ */
public boolean isIngestRunning() { public boolean isIngestRunning() {
return (ingestJobs.isEmpty() == false); return scheduler.ingestJobsAreRunning();
} }
void addFileToIngestJob(long ingestJobId, AbstractFile file) { public void cancelAllIngestJobs() {
IngestJob job = ingestJobs.get(ingestJobId); // Stop creating new ingest jobs.
if (job != null) { for (Future<Void> handle : startIngestJobThreads.values()) {
scheduler.getFileIngestScheduler().queueFile(job, file); handle.cancel(true);
try {
// Blocks until the job starting thread responds. The thread
// removes itself from this collection, which does not disrupt
// this loop since the collection is a ConcurrentHashMap.
handle.get();
} catch (InterruptedException | ExecutionException ex) {
// This should never happen, something is awry, but everything
// should be o.k. anyway.
logger.log(Level.SEVERE, "Unexpected thread interrupt", ex);
} }
} }
void cancelIngestJobs() { // Cancel all the jobs already created. This will make the the ingest
new IngestCancellationWorker().execute(); // threads flush out any lingering ingest tasks without processing them.
scheduler.cancelAllIngestJobs();
} }
/** /**
@ -144,20 +230,21 @@ public class IngestManager {
public enum IngestEvent { public enum IngestEvent {
/** /**
* Property change event fired when an ingest job is started. The ingest * Property change event fired when an ingest job is started. The old
* job id is in old value field of the PropertyChangeEvent object. * value of the PropertyChangeEvent object is set to the ingest job id,
* and the new value is set to null.
*/ */
INGEST_JOB_STARTED, INGEST_JOB_STARTED,
/** /**
* Property change event fired when an ingest job is completed. The * Property change event fired when an ingest job is completed. The old
* ingest job id is in old value field of the PropertyChangeEvent * value of the PropertyChangeEvent object is set to the ingest job id,
* object. * and the new value is set to null.
*/ */
INGEST_JOB_COMPLETED, INGEST_JOB_COMPLETED,
/** /**
* Property change event fired when an ingest job is canceled. The * Property change event fired when an ingest job is canceled. The old
* ingest job id is in old value field of the PropertyChangeEvent * value of the PropertyChangeEvent object is set to the ingest job id,
* object. * and the new value is set to null.
*/ */
INGEST_JOB_CANCELLED, INGEST_JOB_CANCELLED,
/** /**
@ -182,79 +269,122 @@ public class IngestManager {
}; };
/** /**
* Add property change listener to listen to ingest events. * Add an ingest job event property change listener.
* *
* @param listener PropertyChangeListener to register * @param listener The PropertyChangeListener to register.
*/
public void addIngestJobEventListener(final PropertyChangeListener listener) {
ingestJobEventPublisher.addPropertyChangeListener(listener);
}
/**
* Remove an ingest job event property change listener.
*
* @param listener The PropertyChangeListener to unregister.
*/
public void removeIngestJobEventListener(final PropertyChangeListener listener) {
ingestJobEventPublisher.removePropertyChangeListener(listener);
}
/**
* Add an ingest module event property change listener.
*
* @param listener The PropertyChangeListener to register.
*/
public void addIngestModuleEventListener(final PropertyChangeListener listener) {
ingestModuleEventPublisher.addPropertyChangeListener(listener);
}
/**
* Remove an ingest module event property change listener.
*
* @param listener The PropertyChangeListener to unregister.
*/
public void removeIngestModuleEventListener(final PropertyChangeListener listener) {
ingestModuleEventPublisher.removePropertyChangeListener(listener);
}
/**
* Add an ingest module event property change listener.
*
* @deprecated
* @param listener The PropertyChangeListener to register.
*/ */
public static void addPropertyChangeListener(final PropertyChangeListener listener) { public static void addPropertyChangeListener(final PropertyChangeListener listener) {
pcs.addPropertyChangeListener(listener); instance.ingestModuleEventPublisher.addPropertyChangeListener(listener);
} }
/**
* Remove an ingest module event property change listener.
*
* @deprecated
* @param listener The PropertyChangeListener to unregister.
*/
public static void removePropertyChangeListener(final PropertyChangeListener listener) { public static void removePropertyChangeListener(final PropertyChangeListener listener) {
pcs.removePropertyChangeListener(listener); instance.ingestModuleEventPublisher.removePropertyChangeListener(listener);
}
static void fireIngestJobEvent(String eventType, long jobId) {
try {
pcs.firePropertyChange(eventType, jobId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event when file is done with a pipeline run * Fire an ingest event signifying an ingest job started.
* *
* @param fileId ID of file that is done * @param ingestJobId The ingest job id.
*/ */
static void fireFileIngestDone(long fileId) { void fireIngestJobStarted(long ingestJobId) {
try { fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_STARTED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event for ModuleDataEvent (when modules post data to blackboard, * Fire an ingest event signifying an ingest job finished.
* etc.)
* *
* @param moduleDataEvent * @param ingestJobId The ingest job id.
*/ */
static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { void fireIngestJobCompleted(long ingestJobId) {
try { fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event for ModuleContentChanged (when modules create new content that * Fire an ingest event signifying an ingest job was canceled.
* needs to be analyzed)
* *
* @param moduleContentEvent * @param ingestJobId The ingest job id.
*/ */
static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { void fireIngestJobCancelled(long ingestJobId) {
try { fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/**
* Fire an ingest event signifying the ingest of a file is completed.
*
* @param fileId The object id of file.
*/
void fireFileIngestDone(long fileId) {
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.FILE_DONE, fileId, null));
}
/**
* Fire an event signifying a blackboard post by an ingest module.
*
* @param moduleDataEvent A ModuleDataEvent with the details of the posting.
*/
void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) {
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.DATA, moduleDataEvent, null));
}
/**
* Fire an event signifying discovery of additional content by an ingest
* module.
*
* @param moduleDataEvent A ModuleContentEvent with the details of the new
* content.
*/
void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) {
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.CONTENT_CHANGED, moduleContentEvent, null));
}
/**
* Post a message to the ingest messages in box.
*
* @param message The message to be posted.
*/
void postIngestMessage(IngestMessage message) { void postIngestMessage(IngestMessage message) {
if (ingestMessageBox != null) { if (ingestMessageBox != null) {
ingestMessageBox.displayMessage(message); ingestMessageBox.displayMessage(message);
@ -262,11 +392,10 @@ public class IngestManager {
} }
/** /**
* Get free disk space of a drive where ingest data are written to That * Get the free disk space of the drive where to which ingest data is being
* drive is being monitored by IngestMonitor thread when ingest is running. * written, as reported by the ingest monitor.
* Use this method to get amount of free disk space anytime.
* *
* @return amount of disk space, -1 if unknown * @return Free disk space, -1 if unknown
*/ */
long getFreeDiskSpace() { long getFreeDiskSpace() {
if (ingestMonitor != null) { if (ingestMonitor != null) {
@ -276,40 +405,26 @@ public class IngestManager {
} }
} }
private void reportRunIngestModulesTaskDone(long taskId) { /**
ingestTasks.remove(taskId); * Creates ingest jobs.
*/
private class StartIngestJobsThread implements Callable<Void> {
List<Long> completedJobs = new ArrayList<>(); private final long threadId;
for (IngestJob job : ingestJobs.values()) {
job.releaseIngestPipelinesForThread(taskId);
if (job.areIngestPipelinesShutDown() == true) {
completedJobs.add(job.getId());
}
}
for (Long jobId : completedJobs) {
IngestJob job = ingestJobs.remove(jobId);
fireEventTasksExecutor.submit(new FireIngestJobEventTask(jobId, job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED : IngestEvent.INGEST_JOB_COMPLETED));
}
}
private class StartIngestJobsTask implements Runnable {
private final long id;
private final List<Content> dataSources; private final List<Content> dataSources;
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private final boolean processUnallocatedSpace; private final boolean processUnallocatedSpace;
private ProgressHandle progress; private ProgressHandle progress;
StartIngestJobsTask(long taskId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) { StartIngestJobsThread(long threadId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
this.id = taskId; this.threadId = threadId;
this.dataSources = dataSources; this.dataSources = dataSources;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace; this.processUnallocatedSpace = processUnallocatedSpace;
} }
@Override @Override
public void run() { public Void call() {
try { try {
final String displayName = NbBundle.getMessage(this.getClass(), final String displayName = NbBundle.getMessage(this.getClass(),
"IngestManager.StartIngestJobsTask.run.displayName"); "IngestManager.StartIngestJobsTask.run.displayName");
@ -321,33 +436,29 @@ public class IngestManager {
"IngestManager.StartIngestJobsTask.run.cancelling", "IngestManager.StartIngestJobsTask.run.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); cancelFileIngestThread(threadId);
return true; return true;
} }
}); });
progress.start(dataSources.size());
progress.start(2 * dataSources.size()); if (!ingestMonitor.isRunning()) {
int workUnitsCompleted = 0; ingestMonitor.start();
}
int dataSourceProcessed = 0;
for (Content dataSource : dataSources) { for (Content dataSource : dataSources) {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
break; break;
} }
// Create an ingest job. // Start an ingest job for the data source.
IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace); List<IngestModuleError> errors = scheduler.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace);
ingestJobs.put(ingestJob.getId(), ingestJob);
// Start at least one instance of each kind of ingest
// pipeline for this ingest job. This allows for an early out
// if the full ingest module lineup specified by the user
// cannot be started up.
List<IngestModuleError> errors = ingestJob.startUpIngestPipelines();
if (!errors.isEmpty()) { if (!errors.isEmpty()) {
// Report the error to the user. // Report the errors to the user. They have already been logged.
StringBuilder moduleStartUpErrors = new StringBuilder(); StringBuilder moduleStartUpErrors = new StringBuilder();
for (IngestModuleError error : errors) { for (IngestModuleError error : errors) {
String moduleName = error.getModuleDisplayName(); String moduleName = error.getModuleDisplayName();
logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); //NON-NLS
moduleStartUpErrors.append(moduleName); moduleStartUpErrors.append(moduleName);
moduleStartUpErrors.append(": "); moduleStartUpErrors.append(": ");
moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage());
@ -367,169 +478,75 @@ public class IngestManager {
JOptionPane.showMessageDialog(null, notifyMessage.toString(), JOptionPane.showMessageDialog(null, notifyMessage.toString(),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(),
"IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE);
// Jettison the ingest job and move on to the next one.
ingestJob.cancel();
ingestJobs.remove(ingestJob.getId());
break;
} }
progress.progress(++dataSourceProcessed);
// Queue the data source ingest tasks for the ingest job.
final String inputName = dataSource.getName();
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg1",
inputName), workUnitsCompleted);
scheduler.getDataSourceIngestScheduler().queueForIngest(ingestJob);
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg2",
inputName), ++workUnitsCompleted);
// Queue the file ingest tasks for the ingest job.
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg3",
inputName), workUnitsCompleted);
scheduler.getFileIngestScheduler().queueForIngest(ingestJob);
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg4",
inputName), ++workUnitsCompleted);
if (!Thread.currentThread().isInterrupted()) { if (!Thread.currentThread().isInterrupted()) {
if (!ingestMonitor.isRunning()) { break;
ingestMonitor.start();
}
long taskId = ingestTaskId.incrementAndGet();
Future<?> task = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
int numberOfFileTasksRequested = getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfFileTasksRequested; ++i) {
taskId = ingestTaskId.incrementAndGet();
task = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
}
fireEventTasksExecutor.submit(new FireIngestJobEventTask(ingestJob.getId(), IngestEvent.INGEST_JOB_STARTED));
} }
} }
} catch (Exception ex) {
String message = String.format("StartIngestJobsTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
MessageNotifyUtil.Message.error(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg"));
} finally { } finally {
progress.finish(); progress.finish();
ingestTasks.remove(id); startIngestJobThreads.remove(threadId);
}
}
}
private class RunDataSourceIngestModulesTask implements Runnable {
private final long id;
RunDataSourceIngestModulesTask(long taskId) {
id = taskId;
}
@Override
public void run() {
try {
IngestScheduler.DataSourceIngestScheduler scheduler = IngestScheduler.getInstance().getDataSourceIngestScheduler();
IngestJob job = scheduler.getNextTask();
while (job != null) {
if (Thread.currentThread().isInterrupted()) {
break;
}
job.getDataSourceIngestPipelineForThread(id).process();
job = scheduler.getNextTask();
}
} catch (Exception ex) {
String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
}
}
}
private class RunFileSourceIngestModulesTask implements Runnable {
private final long id;
RunFileSourceIngestModulesTask(long taskId) {
id = taskId;
}
@Override
public void run() {
try {
IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler();
FileIngestTask task = fileScheduler.getNextTask();
while (task != null) {
if (Thread.currentThread().isInterrupted()) {
break;
}
IngestJob job = task.getJob();
job.updateFileTasksProgressBar(task.getFile().getName());
job.getFileIngestPipelineForThread(id).process(task.getFile());
task = fileScheduler.getNextTask();
}
} catch (Exception ex) {
String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
}
}
}
private class FireIngestJobEventTask implements Runnable {
private final long ingestJobId;
private final IngestEvent event;
FireIngestJobEventTask(long ingestJobId, IngestEvent event) {
this.ingestJobId = ingestJobId;
this.event = event;
}
@Override
public void run() {
fireIngestJobEvent(event.toString(), ingestJobId);
}
}
private class IngestCancellationWorker extends SwingWorker<Void, Void> {
@Override
protected Void doInBackground() throws Exception {
// First mark all of the ingest jobs as cancelled. This way the
// ingest modules will know they are being shut down due to
// cancellation when the cancelled run ingest module tasks release
// their pipelines.
for (IngestJob job : ingestJobs.values()) {
job.cancel();
}
for (Future<?> task : ingestTasks.values()) {
task.cancel(true);
}
// Jettision the remaining data source and file ingest tasks.
scheduler.getFileIngestScheduler().emptyQueues();
scheduler.getDataSourceIngestScheduler().emptyQueues();
return null; return null;
} }
}
}
/**
* A consumer for an ingest task queue.
*/
private class ExecuteIngestTasksThread implements Runnable {
private IngestTaskQueue tasks;
ExecuteIngestTasksThread(IngestTaskQueue tasks) {
this.tasks = tasks;
}
@Override @Override
protected void done() { public void run() {
while (true) {
try { try {
super.get(); IngestTask task = tasks.getNextTask(); // Blocks.
} catch (CancellationException | InterruptedException ex) { task.execute();
} catch (Exception ex) { scheduler.ingestTaskIsCompleted(task);
logger.log(Level.SEVERE, "Error while cancelling ingest jobs", ex); //NON-NLS } catch (InterruptedException ex) {
break;
}
if (Thread.currentThread().isInterrupted()) {
break;
}
}
}
}
/**
* Fires ingest events to ingest manager property change listeners.
*/
private static class FireIngestEventThread implements Runnable {
private final PropertyChangeSupport publisher;
private final IngestEvent event;
private final Object oldValue;
private final Object newValue;
FireIngestEventThread(PropertyChangeSupport publisher, IngestEvent event, Object oldValue, Object newValue) {
this.publisher = publisher;
this.event = event;
this.oldValue = oldValue;
this.newValue = newValue;
}
@Override
public void run() {
try {
publisher.firePropertyChange(event.toString(), oldValue, newValue);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
} }
} }
} }

2
Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java
View File

@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content;
manager = IngestManager.getInstance(); manager = IngestManager.getInstance();
} }
try { try {
manager.cancelIngestJobs(); manager.cancelAllIngestJobs();
} finally { } finally {
//clear inbox //clear inbox
clearMessages(); clearMessages();

2
Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java
View File

@ -165,7 +165,7 @@ public final class IngestMonitor {
final String diskPath = root.getAbsolutePath(); final String diskPath = root.getAbsolutePath();
MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS
logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS
manager.cancelIngestJobs(); manager.cancelAllIngestJobs();
IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage( IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage(
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath), NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath),
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath))); NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath)));

672
Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java Normal file → Executable file
View File

@ -21,127 +21,118 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.VirtualDirectory;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM;
import org.sleuthkit.datamodel.TskData.TSK_FS_META_TYPE_ENUM;
/**
* Enqueues data source ingest and file ingest tasks for processing.
*/
final class IngestScheduler { final class IngestScheduler {
private static IngestScheduler instance; private static final IngestScheduler instance = new IngestScheduler();
private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName()); private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName());
private final DataSourceIngestScheduler dataSourceIngestScheduler = new DataSourceIngestScheduler(); private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
private final FileIngestScheduler fileIngestScheduler = new FileIngestScheduler(); private final ConcurrentHashMap<Long, IngestJob> ingestJobsById = new ConcurrentHashMap<>();
private final LinkedBlockingQueue<DataSourceIngestTask> dataSourceTasks = new LinkedBlockingQueue<>();
private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); // Guarded by this
private final List<FileIngestTask> directoryTasks = new ArrayList<>(); // Guarded by this
private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>(); // Guarded by this
private final List<IngestTask> tasksInProgress = new ArrayList<>(); // Guarded by this
private final DataSourceIngestTaskQueue dataSourceTaskDispenser = new DataSourceIngestTaskQueue();
private final FileIngestTaskQueue fileTaskDispenser = new FileIngestTaskQueue();
private final AtomicLong nextIngestJobId = new AtomicLong(0L);
static IngestScheduler getInstance() {
return instance;
}
private IngestScheduler() { private IngestScheduler() {
} }
static synchronized IngestScheduler getInstance() { /**
if (instance == null) { * Creates an ingest job for a data source.
instance = new IngestScheduler(); *
* @param rootDataSource The data source to ingest.
* @param ingestModuleTemplates The ingest module templates to use to create
* the ingest pipelines for the job.
* @param processUnallocatedSpace Whether or not the job should include
* processing of unallocated space.
* @return A collection of ingest module start up errors, empty on success.
* @throws InterruptedException
*/
List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException {
long jobId = nextIngestJobId.incrementAndGet();
IngestJob job = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace);
ingestJobsById.put(jobId, job);
IngestManager.getInstance().fireIngestJobStarted(jobId);
List<IngestModuleError> errors = job.startUp();
if (errors.isEmpty()) {
addDataSourceToIngestJob(job, dataSource);
} else {
ingestJobsById.remove(jobId);
IngestManager.getInstance().fireIngestJobCancelled(jobId);
}
return errors;
} }
return instance; boolean ingestJobsAreRunning() {
for (IngestJob job : ingestJobsById.values()) {
if (!job.isCancelled()) {
return true;
}
}
return false;
} }
DataSourceIngestScheduler getDataSourceIngestScheduler() { synchronized void addDataSourceToIngestJob(IngestJob job, Content dataSource) throws InterruptedException {
return dataSourceIngestScheduler; // Enqueue a data source ingest task for the data source.
// If the thread executing this code is interrupted, it is because the
// the number of ingest threads has been decreased while ingest jobs are
// running. The calling thread will exit in an orderly fashion, but the
// task still needs to be enqueued rather than lost, hence the loop.
DataSourceIngestTask task = new DataSourceIngestTask(job, dataSource);
while (true) {
try {
dataSourceTasks.put(task);
break;
} catch (InterruptedException ex) {
// Reset the interrupted status of the thread so the orderly
// exit can occur in the intended place.
Thread.currentThread().interrupt();
}
} }
FileIngestScheduler getFileIngestScheduler() { // Get the top level files of the data source.
return fileIngestScheduler; Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
} List<AbstractFile> toptLevelFiles = new ArrayList<>();
static class FileIngestScheduler {
private TreeSet<FileIngestTask> rootDirectoryTasks;
private List<FileIngestTask> directoryTasks;
private LinkedList<FileIngestTask> fileTasks; //need to add to start and end quickly
private int filesEnqueuedEst = 0;
private int filesDequeued = 0;
private final static int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue()
| TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue()
| TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue()
| TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
private FileIngestScheduler() {
rootDirectoryTasks = new TreeSet<>(new RootTaskComparator());
directoryTasks = new ArrayList<>();
fileTasks = new LinkedList<>();
resetCounters();
}
private void resetCounters() {
filesEnqueuedEst = 0;
filesDequeued = 0;
}
@Override
public synchronized String toString() {
StringBuilder sb = new StringBuilder();
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.rootDirs.text")).append(rootDirectoryTasks.size());
for (FileIngestTask task : rootDirectoryTasks) {
sb.append(task.toString()).append(" ");
}
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curDirs.text")).append(directoryTasks.size());
for (FileIngestTask task : directoryTasks) {
sb.append(task.toString()).append(" ");
}
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curFiles.text")).append(fileTasks.size());
for (FileIngestTask task : fileTasks) {
sb.append(task.toString()).append(" ");
}
return sb.toString();
}
synchronized void queueForIngest(IngestJob dataSourceTask) {
Content dataSource = dataSourceTask.getDataSource();
Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirVisitor());
List<AbstractFile> firstLevelFiles = new ArrayList<>();
if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
// The data source is file. // The data source is itself a file.
firstLevelFiles.add((AbstractFile) dataSource); toptLevelFiles.add((AbstractFile) dataSource);
} else { } else {
for (AbstractFile root : rootObjects) { for (AbstractFile root : rootObjects) {
List<Content> children; List<Content> children;
try { try {
children = root.getChildren(); children = root.getChildren();
if (children.isEmpty()) { if (children.isEmpty()) {
//add the root itself, could be unalloc file, child of volume or image // Add the root object itself, it could be an unallocated space
firstLevelFiles.add(root); // file, or a child of a volume or an image.
toptLevelFiles.add(root);
} else { } else {
//root for fs root dir, schedule children dirs/files // The root object is a file system root directory, get
// the files within it.
for (Content child : children) { for (Content child : children) {
if (child instanceof AbstractFile) { if (child instanceof AbstractFile) {
firstLevelFiles.add((AbstractFile) child); toptLevelFiles.add((AbstractFile) child);
} }
} }
} }
@ -151,197 +142,101 @@ final class IngestScheduler {
} }
} }
for (AbstractFile firstLevelFile : firstLevelFiles) { // Enqueue file ingest tasks for the top level files.
FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask); for (AbstractFile firstLevelFile : toptLevelFiles) {
if (shouldEnqueueTask(fileTask)) { FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile);
if (shouldEnqueueFileTask(fileTask)) {
rootDirectoryTasks.add(fileTask); rootDirectoryTasks.add(fileTask);
} }
} }
// Update approx count of files to process in queues updateFileTaskQueues(null);
filesEnqueuedEst = queryNumFilesinEnqueuedContents();
// Reshuffle/update the dir and file level queues if needed
updateQueues();
} }
synchronized void queueFile(IngestJob ingestJob, AbstractFile file) { void addFileToIngestJob(IngestJob job, AbstractFile file) {
FileIngestTask fileTask = new FileIngestTask(file, ingestJob); FileIngestTask task = new FileIngestTask(job, file);
if (shouldEnqueueTask(fileTask)) { if (shouldEnqueueFileTask(task)) {
fileTasks.addFirst(fileTask); addTaskToFileQueue(task);
++filesEnqueuedEst;
} }
} }
float getPercentageDone() { private synchronized void updateFileTaskQueues(FileIngestTask taskInProgress) throws InterruptedException {
if (filesEnqueuedEst == 0) { if (taskInProgress != null) {
return 0; tasksInProgress.add(taskInProgress);
} }
return ((100.f) * filesDequeued) / filesEnqueuedEst;
}
/**
* query num files enqueued total num of files to be enqueued.
*
* Counts all files for all the sources currently in the queues.
*
* @return approx. total num of files enqueued (or to be enqueued)
*/
private synchronized int queryNumFilesinEnqueuedContents() {
int totalFiles = 0;
List<Content> contents = this.getSourceContent();
final GetFilesCountVisitor countVisitor =
new GetFilesCountVisitor();
for (Content content : contents) {
totalFiles += content.accept(countVisitor);
}
logger.log(Level.INFO, "Total files to queue up: {0}", totalFiles); //NON-NLS
return totalFiles;
}
/**
* get total est. number of files to be enqueued for current ingest
* input sources in queues
*
* @return total number of files
*/
int getFilesEnqueuedEst() {
return filesEnqueuedEst;
}
/**
* Get number of files dequeued so far. This is reset after the same
* content is enqueued that is already in a queue
*
* @return number of files dequeued so far
*/
int getFilesDequeued() {
return filesDequeued;
}
synchronized FileIngestTask getNextTask() {
final FileIngestTask task = fileTasks.pollLast();
if (task != null) {
filesDequeued++;
updateQueues();
}
return task;
}
/**
* Shuffle the queues so that there are files in the files queue.
*
* @returns true if no more data in queue
*/
private synchronized void updateQueues() {
// we loop because we could have a directory that has all files // we loop because we could have a directory that has all files
// that do not get enqueued // that do not get enqueued
while (true) { while (true) {
// There are files in the queue, we're done // There are files in the queue, we're done
if (this.fileTasks.isEmpty() == false) { if (fileTasks.isEmpty() == false) {
return; return;
} }
// fill in the directory queue if it is empty. // fill in the directory queue if it is empty.
if (this.directoryTasks.isEmpty()) { if (this.directoryTasks.isEmpty()) {
// bail out if root is also empty -- we are done // bail out if root is also empty -- we are done
if (rootDirectoryTasks.isEmpty()) { if (rootDirectoryTasks.isEmpty()) {
return; return;
} }
FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst(); FileIngestTask rootTask = rootDirectoryTasks.pollFirst();
directoryTasks.add(rootTask); directoryTasks.add(rootTask);
} }
//pop and push AbstractFile directory children if any //pop and push AbstractFile directory children if any
//add the popped and its leaf children onto cur file list //add the popped and its leaf children onto cur file list
FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1);
final AbstractFile parentFile = parentTask.file; final AbstractFile parentFile = parentTask.getFile();
// add itself to the file list // add itself to the file list
if (shouldEnqueueTask(parentTask)) { if (shouldEnqueueFileTask(parentTask)) {
this.fileTasks.addLast(parentTask); addTaskToFileQueue(parentTask);
} }
// add its children to the file and directory lists // add its children to the file and directory lists
try { try {
List<Content> children = parentFile.getChildren(); List<Content> children = parentFile.getChildren();
for (Content c : children) { for (Content c : children) {
if (c instanceof AbstractFile) { if (c instanceof AbstractFile) {
AbstractFile childFile = (AbstractFile) c; AbstractFile childFile = (AbstractFile) c;
FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getJob()); FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile);
if (childFile.hasChildren()) { if (childFile.hasChildren()) {
this.directoryTasks.add(childTask); directoryTasks.add(childTask);
} else if (shouldEnqueueTask(childTask)) { } else if (shouldEnqueueFileTask(childTask)) {
this.fileTasks.addLast(childTask); addTaskToFileQueue(childTask);
} }
} }
} }
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Could not get children of file and update file queues: " //NON-NLS logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex);
+ parentFile.getName(), ex);
} }
} }
} }
/** private void addTaskToFileQueue(FileIngestTask task) {
* Return list of content objects that are in the queue to be processed. // If the thread executing this code is interrupted, it is because the
* // the number of ingest threads has been decreased while ingest jobs are
* Helpful to determine whether ingest for particular input Content is // running. The calling thread will exit in an orderly fashion, but the
* active // task still needs to be enqueued rather than lost.
* while (true) {
* @return list of parent source content objects for files currently try {
* enqueued fileTasks.put(task);
*/ break;
synchronized List<Content> getSourceContent() { } catch (InterruptedException ex) {
final Set<Content> contentSet = new HashSet<>(); // Reset the interrupted status of the thread so the orderly
// exit can occur in the intended place.
for (FileIngestTask task : rootDirectoryTasks) { Thread.currentThread().interrupt();
contentSet.add(task.getJob().getDataSource());
} }
for (FileIngestTask task : directoryTasks) {
contentSet.add(task.getJob().getDataSource());
} }
for (FileIngestTask task : fileTasks) {
contentSet.add(task.getJob().getDataSource());
} }
return new ArrayList<>(contentSet); private static boolean shouldEnqueueFileTask(final FileIngestTask processTask) {
} final AbstractFile aFile = processTask.getFile();
synchronized void emptyQueues() {
this.rootDirectoryTasks.clear();
this.directoryTasks.clear();
this.fileTasks.clear();
}
/**
* Check if the file is a special file that we should skip
*
* @param processTask a task whose file to check if should be queued of
* skipped
* @return true if should be enqueued, false otherwise
*/
private static boolean shouldEnqueueTask(final FileIngestTask processTask) {
final AbstractFile aFile = processTask.file;
//if it's unalloc file, skip if so scheduled //if it's unalloc file, skip if so scheduled
if (processTask.getJob().shouldProcessUnallocatedSpace() == false if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
&& aFile.getType().equals(TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS //unalloc files
)) {
return false; return false;
} }
String fileName = aFile.getName(); String fileName = aFile.getName();
if (fileName.equals(".") || fileName.equals("..")) { if (fileName.equals(".") || fileName.equals("..")) {
return false; return false;
} else if (aFile instanceof org.sleuthkit.datamodel.File) { } else if (aFile instanceof org.sleuthkit.datamodel.File) {
final org.sleuthkit.datamodel.File f = (File) aFile; final org.sleuthkit.datamodel.File f = (File) aFile;
//skip files in root dir, starting with $, containing : (not default attributes) //skip files in root dir, starting with $, containing : (not default attributes)
//with meta address < 32, i.e. some special large NTFS and FAT files //with meta address < 32, i.e. some special large NTFS and FAT files
FileSystem fs = null; FileSystem fs = null;
@ -354,156 +249,125 @@ final class IngestScheduler {
if (fs != null) { if (fs != null) {
fsType = fs.getFsType(); fsType = fs.getFsType();
} }
if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) {
//not fat or ntfs, accept all files //not fat or ntfs, accept all files
return true; return true;
} }
boolean isInRootDir = false; boolean isInRootDir = false;
try { try {
isInRootDir = f.getParentDirectory().isRoot(); isInRootDir = f.getParentDirectory().isRoot();
} catch (TskCoreException ex) { } catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS
} }
if (isInRootDir && f.getMetaAddr() < 32) { if (isInRootDir && f.getMetaAddr() < 32) {
String name = f.getName(); String name = f.getName();
if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) {
if (name.length() > 0
&& name.charAt(0) == '$'
&& name.contains(":")) {
return false; return false;
} }
} else { } else {
return true; return true;
} }
}
return true;
}
static class FileIngestTask {
private final AbstractFile file;
private final IngestJob task;
private FileIngestTask(AbstractFile file, IngestJob task) {
this.file = file;
this.task = task;
}
public IngestJob getJob() {
return task;
}
public AbstractFile getFile() {
return file;
}
@Override
public String toString() {
try {
return "ProcessTask{" + "file=" + file.getId() + ": " //NON-NLS
+ file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}';
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS
}
return "ProcessTask{" + "file=" + file.getId() + ": " //NON-NLS
+ file.getName() + '}';
}
/**
* two process tasks are equal when the file/dir and modules are the
* same this enables are not to queue up the same file/dir, modules
* tuples into the root dir set
*
* @param obj
* @return
*/
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final FileIngestTask other = (FileIngestTask) obj;
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
return false;
}
IngestJob thisTask = this.getJob();
IngestJob otherTask = other.getJob();
if (thisTask != otherTask
&& (thisTask == null || !thisTask.equals(otherTask))) {
return false;
} }
return true; return true;
} }
@Override void cancelAllIngestJobs() {
public int hashCode() { for (IngestJob job : ingestJobsById.values()) {
int hash = 5; job.cancel();
hash = 47 * hash + Objects.hashCode(this.file);
hash = 47 * hash + Objects.hashCode(this.task);
return hash;
} }
} }
/** IngestTaskQueue getDataSourceIngestTaskQueue() {
* Root dir sorter return dataSourceTaskDispenser;
*/ }
private static class RootTaskComparator implements Comparator<FileIngestTask> {
IngestTaskQueue getFileIngestTaskQueue() {
return fileTaskDispenser;
}
void ingestTaskIsCompleted(IngestTask completedTask) {
if (ingestJobIsCompleted(completedTask)) {
IngestJob job = completedTask.getIngestJob();
job.shutDown();
ingestJobsById.remove(job.getId());
IngestManager.getInstance().fireIngestJobCompleted(job.getId());
}
}
private synchronized boolean ingestJobIsCompleted(IngestTask completedTask) {
tasksInProgress.remove(completedTask);
IngestJob job = completedTask.getIngestJob();
long jobId = job.getId();
for (IngestTask task : tasksInProgress) {
if (task.getIngestJob().getId() == jobId) {
return false;
}
}
for (FileIngestTask task : fileTasks) {
if (task.getIngestJob().getId() == jobId) {
return false;
}
}
for (FileIngestTask task : directoryTasks) {
if (task.getIngestJob().getId() == jobId) {
return false;
}
}
for (FileIngestTask task : rootDirectoryTasks) {
if (task.getIngestJob().getId() == jobId) {
return false;
}
}
for (DataSourceIngestTask task : dataSourceTasks) {
if (task.getIngestJob().getId() == jobId) {
return false;
}
}
return true;
}
private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> {
@Override @Override
public int compare(FileIngestTask q1, FileIngestTask q2) { public int compare(FileIngestTask q1, FileIngestTask q2) {
AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file); AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile());
AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file); AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile());
if (p1 == p2) { if (p1 == p2) {
return (int) (q2.file.getId() - q1.file.getId()); return (int) (q2.getFile().getId() - q1.getFile().getId());
} else { } else {
return p2.ordinal() - p1.ordinal(); return p2.ordinal() - p1.ordinal();
} }
} }
/**
* Priority determination for sorted AbstractFile, used by
* RootDirComparator
*/
private static class AbstractFilePriority { private static class AbstractFilePriority {
enum Priority { enum Priority {
LAST, LOW, MEDIUM, HIGH LAST, LOW, MEDIUM, HIGH
}; }
static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>(); static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>();
static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>(); static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>();
static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>(); static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>();
static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>(); static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>();
/* prioritize root directory folders based on the assumption that we are /* prioritize root directory folders based on the assumption that we are
* looking for user content. Other types of investigations may want different * looking for user content. Other types of investigations may want different
* priorities. */ * priorities. */
static {
static /* prioritize root directory folders based on the assumption that we are
* looking for user content. Other types of investigations may want different
* priorities. */ {
// these files have no structure, so they go last // these files have no structure, so they go last
//unalloc files are handled as virtual files in getPriority() //unalloc files are handled as virtual files in getPriority()
//LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
//LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE)); //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE));
LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE)); LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE)); LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
// orphan files are often corrupt and windows does not typically have // orphan files are often corrupt and windows does not typically have
// user content, so put them towards the bottom // user content, so put them towards the bottom
LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE)); LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE)); LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
// all other files go into the medium category too // all other files go into the medium category too
MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE)); MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
// user content is top priority // user content is top priority
HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE)); HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE)); HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
@ -523,222 +387,56 @@ final class IngestScheduler {
//non-fs virtual files and dirs, such as representing unalloc space //non-fs virtual files and dirs, such as representing unalloc space
return AbstractFilePriority.Priority.LAST; return AbstractFilePriority.Priority.LAST;
} }
//determine the fs files priority by name //determine the fs files priority by name
final String path = abstractFile.getName(); final String path = abstractFile.getName();
if (path == null) { if (path == null) {
return AbstractFilePriority.Priority.MEDIUM; return AbstractFilePriority.Priority.MEDIUM;
} }
for (Pattern p : HIGH_PRI_PATHS) { for (Pattern p : HIGH_PRI_PATHS) {
Matcher m = p.matcher(path); Matcher m = p.matcher(path);
if (m.find()) { if (m.find()) {
return AbstractFilePriority.Priority.HIGH; return AbstractFilePriority.Priority.HIGH;
} }
} }
for (Pattern p : MEDIUM_PRI_PATHS) { for (Pattern p : MEDIUM_PRI_PATHS) {
Matcher m = p.matcher(path); Matcher m = p.matcher(path);
if (m.find()) { if (m.find()) {
return AbstractFilePriority.Priority.MEDIUM; return AbstractFilePriority.Priority.MEDIUM;
} }
} }
for (Pattern p : LOW_PRI_PATHS) { for (Pattern p : LOW_PRI_PATHS) {
Matcher m = p.matcher(path); Matcher m = p.matcher(path);
if (m.find()) { if (m.find()) {
return AbstractFilePriority.Priority.LOW; return AbstractFilePriority.Priority.LOW;
} }
} }
for (Pattern p : LAST_PRI_PATHS) { for (Pattern p : LAST_PRI_PATHS) {
Matcher m = p.matcher(path); Matcher m = p.matcher(path);
if (m.find()) { if (m.find()) {
return AbstractFilePriority.Priority.LAST; return AbstractFilePriority.Priority.LAST;
} }
} }
//default is medium //default is medium
return AbstractFilePriority.Priority.MEDIUM; return AbstractFilePriority.Priority.MEDIUM;
} }
} }
} }
/** private class DataSourceIngestTaskQueue implements IngestTaskQueue {
* Get counts of ingestable files/dirs for the content input source.
*
* Note, also includes counts of all unalloc children files (for the fs,
* image, volume) even if ingest didn't ask for them
*/
static class GetFilesCountVisitor extends ContentVisitor.Default<Long> {
@Override @Override
public Long visit(FileSystem fs) { public IngestTask getNextTask() throws InterruptedException {
//recursion stop here return dataSourceTasks.take();
//case of a real fs, query all files for it
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
StringBuilder queryB = new StringBuilder();
queryB.append("( (fs_obj_id = ").append(fs.getId()); //NON-NLS
//queryB.append(") OR (fs_obj_id = NULL) )");
queryB.append(") )");
queryB.append(" AND ( (meta_type = ").append(TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS
queryB.append(") OR (meta_type = ").append(TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS
queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS
queryB.append(") )");
//queryB.append( "AND (type = ");
//queryB.append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.getFileType());
//queryB.append(")");
try {
final String query = queryB.toString();
logger.log(Level.INFO, "Executing count files query: {0}", query); //NON-NLS
return sc.countFilesWhere(query);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); //NON-NLS
return 0L;
} }
} }
private class FileIngestTaskQueue implements IngestTaskQueue {
@Override @Override
public Long visit(LayoutFile lf) { public IngestTask getNextTask() throws InterruptedException {
//recursion stop here FileIngestTask task = fileTasks.take();
//case of LayoutFile child of Image or Volume updateFileTaskQueues(task);
return 1L; return task;
}
private long getCountFromChildren(Content content) {
long count = 0;
try {
List<Content> children = content.getChildren();
if (children.size() > 0) {
for (Content child : children) {
count += child.accept(this);
}
} else {
count = 1;
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not get count of objects from children to get num of total files to be ingested", ex); //NON-NLS
}
return count;
}
@Override
protected Long defaultVisit(Content cntnt) {
//recurse assuming this is image/vs/volume
//recursion stops at fs or unalloc file
return getCountFromChildren(cntnt);
}
}
/**
* Visitor that gets a collection of top level objects to be scheduled,
* such as root Dirs (if there is FS) or LayoutFiles and virtual
* directories, also if there is no FS.
*/
static class GetRootDirVisitor extends GetFilesContentVisitor {
@Override
public Collection<AbstractFile> visit(VirtualDirectory ld) {
//case when we hit a layout directoryor local file container, not under a real FS
//or when root virt dir is scheduled
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(ld);
return ret;
}
@Override
public Collection<AbstractFile> visit(LayoutFile lf) {
//case when we hit a layout file, not under a real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(lf);
return ret;
}
@Override
public Collection<AbstractFile> visit(Directory drctr) {
//we hit a real directory, a child of real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(drctr);
return ret;
}
@Override
public Collection<AbstractFile> visit(FileSystem fs) {
return getAllFromChildren(fs);
}
@Override
public Collection<AbstractFile> visit(File file) {
//can have derived files
return getAllFromChildren(file);
}
@Override
public Collection<AbstractFile> visit(DerivedFile derivedFile) {
//can have derived files
//TODO test this and overall scheduler with derived files
return getAllFromChildren(derivedFile);
}
@Override
public Collection<AbstractFile> visit(LocalFile localFile) {
//can have local files
//TODO test this and overall scheduler with local files
return getAllFromChildren(localFile);
}
}
}
static class DataSourceIngestScheduler {
private final LinkedList<IngestJob> tasks = new LinkedList<>();
private DataSourceIngestScheduler() {
}
synchronized void queueForIngest(IngestJob job) {
try {
if (job.getDataSource().getParent() != null) {
logger.log(Level.SEVERE, "Only parent-less Content (data sources) can be scheduled for DataSource ingest, skipping: {0}", job.getDataSource()); //NON-NLS
return;
}
} catch (TskCoreException e) {
logger.log(Level.SEVERE, "Error validating data source to be scheduled for DataSource ingest" + job.getDataSource(), e); //NON-NLS
return;
}
tasks.addLast(job);
}
public synchronized IngestJob getNextTask() {
return tasks.pollFirst();
}
synchronized void emptyQueues() {
tasks.clear();
}
synchronized int getCount() {
return tasks.size();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.DataSourceScheduler.toString.size"))
.append(getCount());
for (IngestJob task : tasks) {
sb.append(task.toString()).append(" ");
}
return sb.toString();
} }
} }
} }

4
Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java
View File

@ -95,7 +95,7 @@ public final class IngestServices {
* artifact data * artifact data
*/ */
public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
IngestManager.fireModuleDataEvent(moduleDataEvent); IngestManager.getInstance().fireIngestModuleDataEvent(moduleDataEvent);
} }
/** /**
@ -107,7 +107,7 @@ public final class IngestServices {
* changed * changed
*/ */
public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
IngestManager.fireModuleContentEvent(moduleContentEvent); IngestManager.getInstance().fireIngestModuleContentEvent(moduleContentEvent);
} }
/** /**

34
Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java Executable file
View File

@ -0,0 +1,34 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
abstract class IngestTask {
private final IngestJob job;
IngestTask(IngestJob job) {
this.job = job;
}
IngestJob getIngestJob() {
return job;
}
abstract void execute() throws InterruptedException;
}

23
Core/src/org/sleuthkit/autopsy/ingest/IngestTaskQueue.java Executable file
View File

@ -0,0 +1,23 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
interface IngestTaskQueue {
IngestTask getNextTask() throws InterruptedException;
}

4
Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java
View File

@ -45,11 +45,11 @@ public final class RunIngestModulesDialog extends JDialog {
private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestDialog.title.text"); private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestDialog.title.text");
private static Dimension DIMENSIONS = new Dimension(500, 300); private static Dimension DIMENSIONS = new Dimension(500, 300);
private List<Content> dataSources = new ArrayList<>(); private List<Content> dataSources = new ArrayList<>();
private IngestJobLauncher ingestJobLauncher; private IngestJobConfigurator ingestJobLauncher;
public RunIngestModulesDialog(JFrame frame, String title, boolean modal) { public RunIngestModulesDialog(JFrame frame, String title, boolean modal) {
super(frame, title, modal); super(frame, title, modal);
ingestJobLauncher = new IngestJobLauncher(RunIngestModulesDialog.class.getCanonicalName()); ingestJobLauncher = new IngestJobConfigurator(RunIngestModulesDialog.class.getCanonicalName());
List<String> messages = ingestJobLauncher.getIngestJobConfigWarnings(); List<String> messages = ingestJobLauncher.getIngestJobConfigWarnings();
if (messages.isEmpty() == false) { if (messages.isEmpty() == false) {
StringBuilder warning = new StringBuilder(); StringBuilder warning = new StringBuilder();

2
HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java
View File

@ -32,7 +32,7 @@ import org.openide.util.NbBundle;
import org.openide.util.Utilities; import org.openide.util.Utilities;
import org.openide.util.Lookup; import org.openide.util.Lookup;
import org.openide.util.actions.Presenter; import org.openide.util.actions.Presenter;
import org.sleuthkit.autopsy.ingest.IngestJobLauncher; import org.sleuthkit.autopsy.ingest.IngestJobConfigurator;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskCoreException;

2
HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java
View File

@ -69,7 +69,7 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan
// Listen to the ingest modules to refresh the enabled/disabled state of // Listen to the ingest modules to refresh the enabled/disabled state of
// the components in sync with file ingest. // the components in sync with file ingest.
IngestManager.addPropertyChangeListener(new PropertyChangeListener() { IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
if (isIngestJobEvent(evt)) { if (isIngestJobEvent(evt)) {

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java
View File

@ -126,7 +126,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
setButtonStates(); setButtonStates();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() { IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName(); String changed = evt.getPropertyName();

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java
View File

@ -118,7 +118,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
ingestRunning = IngestManager.getInstance().isIngestRunning(); ingestRunning = IngestManager.getInstance().isIngestRunning();
updateComponents(); updateComponents();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() { IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override @Override
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName(); String changed = evt.getPropertyName();

0
c:casesSamll2Againautopsy.db Executable file
View File