mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-12 16:06:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/sleuthkit/autopsy into core-filesearch

Browse Source
This commit is contained in:
Nick Davis 2014-04-14 16:25:09 -04:00
commit 84609fc144
180 changed files with 4054 additions and 4080 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Core/nbproject/project.properties
View File

@ -1,5 +1,10 @@
file.reference.jdom-2.0.5-contrib.jar=release/modules/ext/jdom-2.0.5-contrib.jar
file.reference.jdom-2.0.5.jar=release/modules/ext/jdom-2.0.5.jar
file.reference.metadata-extractor-2.6.2.jar=release/modules/ext/metadata-extractor-2.6.2.jar
file.reference.sqlite-jdbc-3.8.0-SNAPSHOT.jar=release/modules/ext/sqlite-jdbc-3.8.0-SNAPSHOT.jar
file.reference.tika-core-1.2.jar=release/modules/ext/tika-core-1.2.jar
file.reference.Tsk_DataModel.jar=release/modules/ext/Tsk_DataModel.jar
file.reference.xmpcore.jar=release/modules/ext/xmpcore.jar
javac.source=1.7
javac.compilerargs=-Xlint -Xlint:-serial
license.file=../LICENSE-2.0.txt

28
Core/nbproject/project.xml
View File

@ -207,6 +207,26 @@
<package>org.sleuthkit.autopsy.report</package>
<package>org.sleuthkit.datamodel</package>
</public-packages>
<class-path-extension>
<runtime-relative-path>ext/jdom-2.0.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jdom-2.0.5.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sevenzipjbinding.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sevenzipjbinding.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/xmpcore.jar</runtime-relative-path>
<binary-origin>release/modules/ext/xmpcore.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jdom-2.0.5-contrib.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jdom-2.0.5-contrib.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sevenzipjbinding-AllPlatforms.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sevenzipjbinding-AllPlatforms.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/Tsk_DataModel.jar</runtime-relative-path>
<binary-origin>release/modules/ext/Tsk_DataModel.jar</binary-origin>
@ -215,6 +235,14 @@
<runtime-relative-path>ext/sqlite-jdbc-3.8.0-SNAPSHOT.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sqlite-jdbc-3.8.0-SNAPSHOT.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/metadata-extractor-2.6.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/metadata-extractor-2.6.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/tika-core-1.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/tika-core-1.2.jar</binary-origin>
</class-path-extension>
</data>
</configuration>
</project>

0
Timeline/release/mactime/mactime.exe → Core/release/mactime/mactime.exe
View File

0
ExifParser/release/modules/ext/LICENSE-2.0.txt → Core/release/modules/ext/LICENSE-2.0.txt
View File

0
ExifParser/release/modules/ext/README.txt → Core/release/modules/ext/README.txt
View File

BIN
Core/release/modules/ext/jdom-2.0.5-contrib.jar Normal file
View File

Binary file not shown.

BIN
Core/release/modules/ext/jdom-2.0.5.jar Normal file
View File

Binary file not shown.

0
ExifParser/release/modules/ext/metadata-extractor-2.6.2.jar → Core/release/modules/ext/metadata-extractor-2.6.2.jar Normal file → Executable file
View File

0
SevenZip/release/modules/ext/sevenzipjbinding-AllPlatforms.jar → Core/release/modules/ext/sevenzipjbinding-AllPlatforms.jar Normal file → Executable file
View File

0
SevenZip/release/modules/ext/sevenzipjbinding.jar → Core/release/modules/ext/sevenzipjbinding.jar Normal file → Executable file
View File

0
FileTypeId/release/modules/ext/tika-core-1.2.jar → Core/release/modules/ext/tika-core-1.2.jar Normal file → Executable file
View File

0
ExifParser/release/modules/ext/xmpcore.jar → Core/release/modules/ext/xmpcore.jar Normal file → Executable file
View File

1
Core/src/org/sleuthkit/autopsy/actions/Bundle.properties
View File

@ -48,3 +48,4 @@ GetTagNameDialog.unableToAddTagNameToCase.msg=Unable to add the {0} tag name to
GetTagNameDialog.taggingErr=Tagging Error
GetTagNameDialog.tagNameAlreadyDef.msg=A {0} tag name has already been defined.
GetTagNameDialog.dupTagErr=Duplicate Tag Error
OpenLogFolder.error1=Log File Not Found: {0}

64
Core/src/org/sleuthkit/autopsy/actions/OpenLogFolderAction.java Executable file
View File

@ -0,0 +1,64 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.actions;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.Desktop;
import java.io.File;
import java.io.IOException;
import org.openide.DialogDisplayer;
import org.openide.NotifyDescriptor;
import org.openide.awt.ActionID;
import org.openide.awt.ActionReference;
import org.openide.awt.ActionRegistration;
import org.openide.modules.Places;
import org.openide.util.Exceptions;
import org.openide.util.NbBundle.Messages;
@ActionID(
category = "Help",
id = "org.sleuthkit.autopsy.actions.OpenLogFolder")
@ActionRegistration(
displayName = "#CTL_OpenLogFolder")
@ActionReference(path = "Menu/Help", position = 1750)
@Messages("CTL_OpenLogFolder=Open Log Folder")
/**
* Action in menu to open the folder containing the log files
*/
public final class OpenLogFolderAction implements ActionListener {
@Override
public void actionPerformed(ActionEvent e) {
try {
File logDir = new File(Places.getUserDirectory().getAbsolutePath() + File.separator + "var" + File.separator + "log");
if (logDir.exists() == false) {
NotifyDescriptor d =
new NotifyDescriptor.Message(
java.text.MessageFormat.format(java.util.ResourceBundle.getBundle("org/sleuthkit/autopsy/actions/Bundle").getString("OpenLogFolder.error1"), new Object[]{logDir.getAbsolutePath()}),
NotifyDescriptor.ERROR_MESSAGE);
DialogDisplayer.getDefault().notify(d);
} else {
Desktop.getDesktop().open(logDir);
}
} catch (IOException ex) {
Exceptions.printStackTrace(ex);
}
}
}

17
Core/src/org/sleuthkit/autopsy/core/layer.xml
View File

@ -122,8 +122,13 @@
</folder>
<folder name="Help">
<file name="org-netbeans-core-actions-AboutAction.instance_hidden"/>
<file name="org-sleuthkit-autopsy-corecomponents-CustomAboutAction.instance">
<attr name="delegate" newvalue="org.sleuthkit.autopsy.corecomponents.CustomAboutAction"/>
<file name="org-sleuthkit-autopsy-actions-OpenLogFolder.instance_hidden"/>
<file name="org-sleuthkit-autopsy-actions-OpenLogFolderAction.instance">
<attr name="instanceCreate" methodvalue="org.openide.awt.Actions.alwaysEnabled"/>
<attr name="noIconInMenu" boolvalue="false"/>
</file>
<file name="org-sleuthkit-autopsy-corecomponents-AboutWindowAction.instance">
<attr name="delegate" newvalue="org.sleuthkit.autopsy.corecomponents.AboutWindowAction"/>
<attr name="displayName" bundlevalue="org.sleuthkit.autopsy.corecomponents.Bundle#CTL_CustomAboutAction"/>
<attr name="instanceCreate" methodvalue="org.openide.awt.Actions.alwaysEnabled"/>
<attr name="noIconInMenu" boolvalue="false"/>
@ -260,7 +265,7 @@
<file name="org-netbeans-modules-autoupdate-ui-actions-CheckForUpdatesAction.shadow_hidden"/>
<attr name="master-help.xml/org-sleuthkit-autopsy-corecomponents-CustomAboutAction.shadow" boolvalue="true"/>
<file name="org-sleuthkit-autopsy-corecomponents-CustomAboutAction.shadow">
<attr name="originalFile" stringvalue="Actions/Help/org-sleuthkit-autopsy-corecomponents-CustomAboutAction.instance"/>
<attr name="originalFile" stringvalue="Actions/Help/org-sleuthkit-autopsy-corecomponents-AboutWindowAction.instance"/>
<attr name="position" intvalue="3000"/>
</file>
</folder>
@ -317,6 +322,12 @@
<attr name="instanceCreate" methodvalue="org.sleuthkit.autopsy.report.FileReportText.getDefault"/>
<attr name="position" intvalue="903"/>
</file>
<file name="org-sleuthkit-autopsy-report-ReportKML.instance">
<attr name="instanceOf" stringvalue="org.sleuthkit.autopsy.report.GeneralReportModule"/>
<attr name="instanceCreate" methodvalue="org.sleuthkit.autopsy.report.ReportKML.getDefault"/>
<attr name="position" intvalue="904"/>
</file>
<!--<folder name="JavaHelp">
<file name="casemodule-helpset.xml" url="casemodule-helpset.xml">
<attr name="position" intvalue="3075"/>

6
Core/src/org/sleuthkit/autopsy/corecomponents/CustomAboutAction.java → Core/src/org/sleuthkit/autopsy/corecomponents/AboutWindowAction.java
View File

@ -28,16 +28,16 @@ import org.sleuthkit.autopsy.coreutils.Logger;
/**
* Action to open custom implementation of the "About" window from the Help menu.
*/
class CustomAboutAction extends AboutAction {
class AboutWindowAction extends AboutAction {
@Override
public void performAction() {
Logger.noteAction(this.getClass());
ProductInformationPanel pip = new ProductInformationPanel();
AboutWindowPanel pip = new AboutWindowPanel();
DialogDescriptor descriptor = new DialogDescriptor(
pip,
NbBundle.getMessage(CustomAboutAction.class, "CTL_CustomAboutAction"),
NbBundle.getMessage(AboutWindowAction.class, "CTL_CustomAboutAction"),
true,
new Object[0],
null,

0
Core/src/org/sleuthkit/autopsy/corecomponents/ProductInformationPanel.form → Core/src/org/sleuthkit/autopsy/corecomponents/AboutWindowPanel.form
View File

32
Core/src/org/sleuthkit/autopsy/corecomponents/ProductInformationPanel.java → Core/src/org/sleuthkit/autopsy/corecomponents/AboutWindowPanel.java
View File

@ -35,7 +35,6 @@ import org.netbeans.core.actions.HTMLViewAction;
import org.openide.awt.HtmlBrowser;
import org.openide.modules.Places;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.datamodel.SleuthkitJNI;
@ -43,17 +42,17 @@ import org.sleuthkit.datamodel.SleuthkitJNI;
/**
* Custom "About" window panel.
*/
public class ProductInformationPanel extends JPanel implements HyperlinkListener {
public class AboutWindowPanel extends JPanel implements HyperlinkListener {
private URL url = null;
private Icon about;
private boolean verboseLogging;
public ProductInformationPanel() {
public AboutWindowPanel() {
about = new ImageIcon(org.netbeans.core.startup.Splash.loadContent(true));
initComponents();
jLabel1.setCursor(Cursor.getPredefinedCursor(Cursor.HAND_CURSOR));
description.setText(org.openide.util.NbBundle.getMessage(ProductInformationPanel.class,
description.setText(org.openide.util.NbBundle.getMessage(AboutWindowPanel.class,
"LBL_Description", new Object[]{getProductVersionValue(), getJavaValue(), getVMValue(),
getOperatingSystemValue(), getEncodingValue(), getSystemLocaleValue(), getUserDirValue(), getSleuthKitVersionValue(), Version.getNetbeansBuild(), Version.getBuildType().toString()}));
description.addHyperlinkListener(this);
@ -96,7 +95,7 @@ public class ProductInformationPanel extends JPanel implements HyperlinkListener
copyright.setBorder(null);
copyright.setContentType("text/html"); // NOI18N
copyright.setEditable(false);
copyright.setText(org.openide.util.NbBundle.getBundle(ProductInformationPanel.class).getString("LBL_Copyright")); // NOI18N
copyright.setText(org.openide.util.NbBundle.getBundle(AboutWindowPanel.class).getString("LBL_Copyright")); // NOI18N
copyright.addMouseListener(new java.awt.event.MouseAdapter() {
public void mouseClicked(java.awt.event.MouseEvent evt) {
copyrightMouseClicked(evt);
@ -109,8 +108,7 @@ public class ProductInformationPanel extends JPanel implements HyperlinkListener
jScrollPane2.setViewportView(description);
verboseLoggingButton.setBackground(new java.awt.Color(255, 255, 255));
verboseLoggingButton.setText(
NbBundle.getMessage(this.getClass(), "ProductInformationPanel.actVerboseLogging.text"));
verboseLoggingButton.setText("Activate verbose logging");
verboseLoggingButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
activateVerboseLogging(evt);
@ -118,7 +116,7 @@ public class ProductInformationPanel extends JPanel implements HyperlinkListener
});
jButton2.setBackground(new java.awt.Color(255, 255, 255));
jButton2.setText(NbBundle.getMessage(ProductInformationPanel.class, "LBL_Close")); // NOI18N
jButton2.setText(NbBundle.getMessage(AboutWindowPanel.class, "LBL_Close")); // NOI18N
jButton2.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
jButton2ActionPerformed(evt);
@ -172,7 +170,7 @@ private void jButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRS
private void jLabel1MouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_jLabel1MouseClicked
try {
url = new URL(NbBundle.getMessage(ProductInformationPanel.class, "URL_ON_IMG")); // NOI18N
url = new URL(NbBundle.getMessage(AboutWindowPanel.class, "URL_ON_IMG")); // NOI18N
showUrl();
} catch (MalformedURLException ex) {
//ignore
@ -226,29 +224,29 @@ private void jLabel1MouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:eve
}
private static String getOperatingSystemValue() {
return NbBundle.getMessage(ProductInformationPanel.class, "Format_OperatingSystem_Value",
return NbBundle.getMessage(AboutWindowPanel.class, "Format_OperatingSystem_Value",
System.getProperty("os.name",
NbBundle.getMessage(ProductInformationPanel.class,
NbBundle.getMessage(AboutWindowPanel.class,
"ProductInformationPanel.propertyUnknown.text")),
System.getProperty("os.version",
NbBundle.getMessage(ProductInformationPanel.class,
NbBundle.getMessage(AboutWindowPanel.class,
"ProductInformationPanel.propertyUnknown.text")),
System.getProperty("os.arch",
NbBundle.getMessage(ProductInformationPanel.class,
NbBundle.getMessage(AboutWindowPanel.class,
"ProductInformationPanel.propertyUnknown.text")));
}
private static String getJavaValue() {
return System.getProperty("java.version",
NbBundle.getMessage(ProductInformationPanel.class,
NbBundle.getMessage(AboutWindowPanel.class,
"ProductInformationPanel.propertyUnknown.text"));
}
private static String getVMValue() {
return NbBundle.getMessage(ProductInformationPanel.class,
return NbBundle.getMessage(AboutWindowPanel.class,
"ProductInformationPanel.getVMValue.text",
System.getProperty("java.vm.name",
NbBundle.getMessage(ProductInformationPanel.class,
NbBundle.getMessage(AboutWindowPanel.class,
"ProductInformationPanel.propertyUnknown.text")),
System.getProperty("java.vm.version", ""));
}
@ -264,7 +262,7 @@ private void jLabel1MouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:eve
private static String getEncodingValue() {
return System.getProperty("file.encoding",
NbBundle.getMessage(ProductInformationPanel.class, "ProductInformationPanel.propertyUnknown.text"));
NbBundle.getMessage(AboutWindowPanel.class, "ProductInformationPanel.propertyUnknown.text"));
}
public void setCopyright(String text) {

5
Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties
View File

@ -1,8 +1,8 @@
CTL_DataContentAction=DataContent
CTL_DataContentTopComponent=Data Content
CTL_CustomAboutAction=About
OptionsCategory_Name_General=Display
OptionsCategory_Keywords_General=display
OptionsCategory_Name_General=Autopsy
OptionsCategory_Keywords_General=Autopsy Options
HINT_DataContentTopComponent=This is a DataContent window
HINT_NodeTableTopComponent=This is a DataResult window
OpenIDE-Module-Name=CoreComponents
@ -147,3 +147,4 @@ DataResultViewerThumbnail.switchPage.done.errMsg=Error making thumbnails\: {0}
FXVideoPanel.pauseButton.infoLabel.playbackErr=Playback error.
GstVideoPanel.progress.infoLabel.updateErr=Error updating video progress\: {0}
GstVideoPanel.ExtractMedia.progress.buffering=Buffering {0}
GeneralPanel.jLabel4.text=Number of threads to use for file ingest:

2
Core/src/org/sleuthkit/autopsy/corecomponents/FrameCapture.java
View File

@ -24,7 +24,7 @@ import java.util.List;
/**
* Interface used to capture frames from a video file.
*/
interface FrameCapture {
public interface FrameCapture {
/**
* @param file the video file to use

2
Core/src/org/sleuthkit/autopsy/corecomponents/GeneralOptionsPanelController.java
View File

@ -30,7 +30,7 @@ import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
@OptionsPanelController.TopLevelRegistration(categoryName = "#OptionsCategory_Name_General",
iconBase = "org/sleuthkit/autopsy/corecomponents/display-options.png",
iconBase = "org/sleuthkit/autopsy/corecomponents/checkbox.png",
position = 1,
keywords = "#OptionsCategory_Keywords_General",
keywordsCategory = "General")

29
Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.form
View File

@ -42,6 +42,7 @@
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jLabel1" min="-2" max="-2" attributes="0"/>
<Component id="jLabel3" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="jLabel4" alignment="0" min="-2" max="-2" attributes="0"/>
<Group type="102" attributes="0">
<EmptySpace min="10" pref="10" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
@ -49,10 +50,11 @@
<Component id="useBestViewerRB" min="-2" max="-2" attributes="0"/>
<Component id="dataSourcesHideKnownCB" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="viewsHideKnownCB" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="numberOfFileIngestThreadsComboBox" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</Group>
<EmptySpace min="0" pref="13" max="32767" attributes="0"/>
<EmptySpace min="0" pref="0" max="32767" attributes="0"/>
</Group>
</Group>
</Group>
@ -78,7 +80,10 @@
<Component id="useLocalTimeRB" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="useGMTTimeRB" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="30" max="32767" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="jLabel4" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="numberOfFileIngestThreadsComboBox" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -173,5 +178,25 @@
</Property>
</Properties>
</Component>
<Component class="javax.swing.JLabel" name="jLabel4">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/corecomponents/Bundle.properties" key="GeneralPanel.jLabel4.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JComboBox" name="numberOfFileIngestThreadsComboBox">
<Properties>
<Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor">
<StringArray count="4">
<StringItem index="0" value="1"/>
<StringItem index="1" value="2"/>
<StringItem index="2" value="3"/>
<StringItem index="3" value="4"/>
</StringArray>
</Property>
<Property name="selectedIndex" type="int" value="1"/>
</Properties>
</Component>
</SubComponents>
</Form>

27
Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2013 Basis Technology Corp.
* Copyright 2013-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -21,10 +21,10 @@ package org.sleuthkit.autopsy.corecomponents;
import java.util.prefs.Preferences;
import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.IngestManager;
final class GeneralPanel extends javax.swing.JPanel {
private final GeneralOptionsPanelController controller;
private static final String KEEP_PREFERRED_VIEWER = "keepPreferredViewer";
private static final String USE_LOCAL_TIME = "useLocalTime";
private static final String DS_HIDE_KNOWN = "dataSourcesHideKnown"; // Default false
@ -32,7 +32,6 @@ final class GeneralPanel extends javax.swing.JPanel {
private final Preferences prefs = NbPreferences.forModule(this.getClass());
GeneralPanel(GeneralOptionsPanelController controller) {
this.controller = controller;
initComponents();
ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected());
// TODO listen to changes in form fields and call controller.changed()
@ -57,6 +56,8 @@ final class GeneralPanel extends javax.swing.JPanel {
jLabel3 = new javax.swing.JLabel();
dataSourcesHideKnownCB = new javax.swing.JCheckBox();
viewsHideKnownCB = new javax.swing.JCheckBox();
jLabel4 = new javax.swing.JLabel();
numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox();
buttonGroup1.add(useBestViewerRB);
useBestViewerRB.setSelected(true);
@ -94,6 +95,11 @@ final class GeneralPanel extends javax.swing.JPanel {
org.openide.awt.Mnemonics.setLocalizedText(viewsHideKnownCB, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.viewsHideKnownCB.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N
numberOfFileIngestThreadsComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "1", "2", "3", "4" }));
numberOfFileIngestThreadsComboBox.setSelectedIndex(1);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
@ -114,14 +120,16 @@ final class GeneralPanel extends javax.swing.JPanel {
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jLabel1)
.addComponent(jLabel3)
.addComponent(jLabel4)
.addGroup(layout.createSequentialGroup()
.addGap(10, 10, 10)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(keepCurrentViewerRB)
.addComponent(useBestViewerRB)
.addComponent(dataSourcesHideKnownCB)
.addComponent(viewsHideKnownCB))))
.addGap(0, 13, Short.MAX_VALUE))))
.addComponent(viewsHideKnownCB)
.addComponent(numberOfFileIngestThreadsComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))))
.addGap(0, 0, Short.MAX_VALUE))))
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@ -143,7 +151,10 @@ final class GeneralPanel extends javax.swing.JPanel {
.addComponent(useLocalTimeRB)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(useGMTTimeRB)
.addContainerGap(30, Short.MAX_VALUE))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(jLabel4)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(numberOfFileIngestThreadsComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
);
}// </editor-fold>//GEN-END:initComponents
@ -164,6 +175,7 @@ final class GeneralPanel extends javax.swing.JPanel {
useGMTTimeRB.setSelected(!useLocalTime);
dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false));
viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true));
numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getInstance().getNumberOfFileIngestThreads());
}
void store() {
@ -171,6 +183,7 @@ final class GeneralPanel extends javax.swing.JPanel {
prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected());
prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected());
prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected());
IngestManager.getInstance().setNumberOfFileIngestThreads(Integer.valueOf(numberOfFileIngestThreadsComboBox.getSelectedItem().toString()));
}
boolean valid() {
@ -184,7 +197,9 @@ final class GeneralPanel extends javax.swing.JPanel {
private javax.swing.JLabel jLabel1;
private javax.swing.JLabel jLabel2;
private javax.swing.JLabel jLabel3;
private javax.swing.JLabel jLabel4;
private javax.swing.JRadioButton keepCurrentViewerRB;
private javax.swing.JComboBox numberOfFileIngestThreadsComboBox;
private javax.swing.JRadioButton useBestViewerRB;
private javax.swing.JRadioButton useGMTTimeRB;
private javax.swing.JRadioButton useLocalTimeRB;

2
Core/src/org/sleuthkit/autopsy/corecomponents/VideoFrame.java
View File

@ -5,7 +5,7 @@ import java.awt.Image;
/**
*
*/
class VideoFrame {
public class VideoFrame {
private Image frame;
private long timeMillis;

BIN
Core/src/org/sleuthkit/autopsy/corecomponents/checkbox.png Executable file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.7 KiB

4
Core/src/org/sleuthkit/autopsy/coreutils/ExecUtil.java
View File

@ -90,7 +90,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
outputStringRedirect = null;
//gc process with its streams
proc = null;
//proc = null;
return output;
}
@ -137,7 +137,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
logger.log(Level.INFO, aCommand + " exit value: " + exitVal);
//gc process with its streams
proc = null;
//proc = null;
}
/**

4
Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java
View File

@ -208,11 +208,11 @@ public class PlatformUtil {
* @throws IOException exception thrown if extract the file failed for IO
* reasons
*/
public static <T> boolean extractResourceToUserConfigDir(final Class<T> resourceClass, final String resourceFile) throws IOException {
public static <T> boolean extractResourceToUserConfigDir(final Class<T> resourceClass, final String resourceFile, boolean overWrite) throws IOException {
final File userDir = new File(getUserConfigDirectory());
final File resourceFileF = new File(userDir + File.separator + resourceFile);
if (resourceFileF.exists()) {
if (resourceFileF.exists() && !overWrite) {
return false;
}

2
Core/src/org/sleuthkit/autopsy/coreutils/XMLUtil.java
View File

@ -70,7 +70,7 @@ public class XMLUtil {
*/
public static <T> boolean xmlIsValid(DOMSource xmlfile, Class<T> clazz, String schemaFile) {
try{
PlatformUtil.extractResourceToUserConfigDir(clazz, schemaFile);
PlatformUtil.extractResourceToUserConfigDir(clazz, schemaFile, false);
File schemaLoc = new File(PlatformUtil.getUserConfigDirectory() + File.separator + schemaFile);
SchemaFactory schm = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
try{

86
Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java
View File

@ -73,6 +73,7 @@ import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskException;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
/**
* Top component which displays something.
*/
@ -82,7 +83,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
private transient ExplorerManager em = new ExplorerManager();
private static DirectoryTreeTopComponent instance;
private DataResultTopComponent dataResult = new DataResultTopComponent(true, NbBundle.getMessage(this.getClass(),
"DirectoryTreeTopComponent.title.text"));
"DirectoryTreeTopComponent.title.text"));
private LinkedList<String[]> backList;
private LinkedList<String[]> forwardList;
/**
@ -222,12 +223,12 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
private void backButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_backButtonActionPerformed
// change the cursor to "waiting cursor" for this operation
this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
// the end is the current place,
String[] currentNodePath = backList.pollLast();
forwardList.addLast(currentNodePath);
forwardButton.setEnabled(true);
/* We peek instead of poll because we use its existence
* in the list later on so that we do not reset the forward list
* after the selection occurs. */
@ -239,7 +240,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
} else {
backButton.setEnabled(false);
}
// update the selection on directory tree
setSelectedNode(newCurrentNodePath, null);
@ -256,10 +257,10 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
} else {
forwardButton.setEnabled(false);
}
backList.addLast(newCurrentNodePath);
backButton.setEnabled(true);
// update the selection on directory tree
setSelectedNode(newCurrentNodePath, null);
@ -543,8 +544,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
// The current case has been closed. Reset the ExplorerManager.
Node emptyNode = new AbstractNode(Children.LEAF);
em.setRootContext(emptyNode);
}
else if (newValue != null) {
} else if (newValue != null) {
// A new case has been opened. Reset the forward and back
// buttons. Note that a call to CoreComponentControl.openCoreWindows()
// by the new Case object will lead to a componentOpened() call
@ -598,7 +598,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
refreshTree(event.getArtifactType());
}
});
} else if (changed.equals(IngestEvent.COMPLETED.toString())) {
} else if (changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
|| changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
@ -654,7 +655,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
return;
}
Node originNode = origin.getNode();
//set node, wrap in filter node first to filter out children
Node drfn = new DataResultFilterNode(originNode, DirectoryTreeTopComponent.this.em);
Node kffn = new KnownFileFilterNode(drfn, KnownFileFilterNode.getSelectionContext(originNode));
@ -667,9 +668,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
displayName = content.getUniquePath();
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Exception while calling Content.getUniquePath() for node: " + originNode);
}
}
else if (originNode.getLookup().lookup(String.class) != null) {
}
} else if (originNode.getLookup().lookup(String.class) != null) {
displayName = originNode.getLookup().lookup(String.class);
}
dataResult.setPath(displayName);
@ -695,12 +695,12 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
// update the back and forward list
updateHistory(em.getSelectedNodes());
}
private void updateHistory(Node[] selectedNodes) {
if (selectedNodes.length == 0) {
return;
}
Node selectedNode = selectedNodes[0];
String selectedNodeName = selectedNode.getName();
@ -729,7 +729,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
forwardButton.setEnabled(false); // disable the forward Button
}
}
/**
* Resets the back and forward list, and also disable the back and forward
* buttons.
@ -752,8 +752,6 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
pcs.removePropertyChangeListener(listener);
}
/**
* Gets the tree on this DirectoryTreeTopComponent.
*
@ -768,13 +766,13 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
*/
public void refreshContentTreeSafe() {
SwingUtilities.invokeLater(new Runnable() {
@Override
public void run() {
refreshContentTree();
}
});
@Override
public void run() {
refreshContentTree();
}
});
}
/**
* Refreshes changed content nodes
*/
@ -865,7 +863,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
/**
* Set the selected node using a path to a previously selected node.
*
* @param previouslySelectedNodePath Path to a previously selected node.
* @param previouslySelectedNodePath Path to a previously selected node.
* @param rootNodeName Name of the root node to match, may be null.
*/
private void setSelectedNode(final String[] previouslySelectedNodePath, final String rootNodeName) {
@ -876,28 +874,26 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
@Override
public void run() {
if (previouslySelectedNodePath.length > 0 && (rootNodeName == null || previouslySelectedNodePath[0].equals(rootNodeName))) {
Node selectedNode = null;
Node selectedNode = null;
ArrayList<String> selectedNodePath = new ArrayList<>(Arrays.asList(previouslySelectedNodePath));
while (null == selectedNode && !selectedNodePath.isEmpty()) {
try {
selectedNode = NodeOp.findPath(em.getRootContext(), selectedNodePath.toArray(new String[0]));
}
catch (NodeNotFoundException ex) {
selectedNode = NodeOp.findPath(em.getRootContext(), selectedNodePath.toArray(new String[0]));
} catch (NodeNotFoundException ex) {
// The selected node may have been deleted (e.g., a deleted tag), so truncate the path and try again.
if (selectedNodePath.size() > 1) {
selectedNodePath.remove(selectedNodePath.size() - 1);
}
else {
} else {
StringBuilder nodePath = new StringBuilder();
for (int i = 0; i < previouslySelectedNodePath.length; ++i) {
nodePath.append(previouslySelectedNodePath[i]).append("/");
}
logger.log(Level.WARNING, "Failed to find any nodes to select on path " + nodePath.toString(), ex);
break;
break;
}
}
}
}
if (null != selectedNode) {
if (rootNodeName != null) {
//called from tree auto refresh context
@ -905,9 +901,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
backList.pollLast();
}
try {
em.setExploredContextAndSelection(selectedNode, new Node[]{selectedNode});
}
catch (PropertyVetoException ex) {
em.setExploredContextAndSelection(selectedNode, new Node[]{selectedNode});
} catch (PropertyVetoException ex) {
logger.log(Level.WARNING, "Property veto from ExplorerManager setting selection to " + selectedNode.getName(), ex);
}
}
@ -970,11 +965,11 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
} catch (TskException ex) {
logger.log(Level.WARNING, "Error retrieving attributes", ex);
}
} else if ( type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT) ||
type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT) ) {
} else if (type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)
|| type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT)) {
Node interestingItemsRootNode = resultsChilds.findChild(type.getLabel());
Children interestingItemsRootChildren = interestingItemsRootNode.getChildren();
try {
try {
String setName = null;
List<BlackboardAttribute> attributes = art.getAttributes();
for (BlackboardAttribute att : attributes) {
@ -1030,16 +1025,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
}
void fireViewerComplete() {
try {
firePropertyChange(BlackboardResultViewer.FINISHED_DISPLAY_EVT, 0, 1);
}
catch (Exception e) {
} catch (Exception e) {
logger.log(Level.SEVERE, "DirectoryTreeTopComponent listener threw exception", e);
MessageNotifyUtil.Notify.show(NbBundle.getMessage(this.getClass(), "DirectoryTreeTopComponent.moduleErr"),
NbBundle.getMessage(this.getClass(),
"DirectoryTreeTopComponent.moduleErr.msg"),
MessageNotifyUtil.MessageType.ERROR);
NbBundle.getMessage(this.getClass(),
"DirectoryTreeTopComponent.moduleErr.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
}
}

126
Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java
View File

@ -35,7 +35,7 @@ import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.casemodule.services.Services;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestModule;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
@ -46,80 +46,42 @@ import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.datamodel.TskData;
/**
* Sample data source ingest module that doesn't do much. Demonstrates per
* ingest job module settings, use of a subset of the available ingest services
* and thread-safe sharing of per ingest job resources.
* <p>
* IMPORTANT TIP: This sample data source ingest module directly implements
* DataSourceIngestModule, which extends IngestModule. A practical alternative,
* recommended if you do not need to provide implementations of all of the
* IngestModule methods, is to extend the abstract class IngestModuleAdapter to
* get default "do nothing" implementations of the IngestModule methods.
* and thread-safe sharing of per ingest job data.
*/
class SampleDataSourceIngestModule implements DataSourceIngestModule {
class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSourceIngestModule {
private static final HashMap<Long, Integer> moduleReferenceCountsForIngestJobs = new HashMap<>();
private static final HashMap<Long, Long> fileCountsForIngestJobs = new HashMap<>();
private static long messageCount = 0;
private final boolean skipKnownFiles;
private IngestJobContext context = null;
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
SampleDataSourceIngestModule(SampleModuleIngestJobSettings settings) {
this.skipKnownFiles = settings.skipKnownFiles();
}
/**
* Invoked by Autopsy to allow an ingest module instance to set up any
* internal data structures and acquire any private resources it will need
* during an ingest job.
* <p>
* Autopsy will generally use several instances of an ingest module for each
* ingest job it performs. Completing an ingest job entails processing a
* single data source (e.g., a disk image) and all of the files from the
* data source, including files extracted from archives and any unallocated
* space (made to look like a series of files). The data source is passed
* through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
* <p>
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that there will be no more than one module instance per
* thread. However, if the module instances must share resources, the
* modules are responsible for synchronizing access to the shared resources
* and doing reference counting as required to release those resources
* correctly. Also, more than one ingest job may be in progress at any given
* time. This must also be taken into consideration when sharing resources
* between module instances.
* <p>
* An ingest module that does not require initialization may extend the
* abstract IngestModuleAdapter class to get a default "do nothing"
* implementation of this method.
*
* @param context Provides data and services specific to the ingest job and
* the ingest pipeline of which the module is a part.
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
*/
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context;
// This method is thread-safe with per ingest job reference counting.
// This method is thread-safe with per ingest job reference counted
// management of shared data.
initFileCount(context.getJobId());
}
/**
* Processes a data source.
*
* @param dataSource The data source to process.
* @param statusHelper A status helper to be used to report progress and
* detect ingest job cancellation.
* @return A result code indicating success or failure of the processing.
*/
@Override
public ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) {
public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar) {
// There are two tasks to do. Set the the progress bar to determinate
// and set the remaining number of work units to be completed to two.
progressBar.switchToDeterminate(2);
Case autopsyCase = Case.getCurrentCase();
SleuthkitCase sleuthkitCase = autopsyCase.getSleuthkitCase();
Services services = new Services(sleuthkitCase);
@ -134,6 +96,8 @@ class SampleDataSourceIngestModule implements DataSourceIngestModule {
}
}
progressBar.progress(1);
// Get files by creation time.
long currentTime = System.currentTimeMillis() / 1000;
long minTime = currentTime - (14 * 24 * 60 * 60); // Go back two weeks.
@ -144,62 +108,33 @@ class SampleDataSourceIngestModule implements DataSourceIngestModule {
}
}
// This method is thread-safe and keeps per ingest job counters.
// This method is thread-safe with per ingest job reference counted
// management of shared data.
addToFileCount(context.getJobId(), fileCount);
progressBar.progress(1);
return IngestModule.ProcessResult.OK;
} catch (TskCoreException ex) {
IngestServices ingestServices = IngestServices.getInstance();
Logger logger = ingestServices.getLogger(SampleIngestModuleFactory.getModuleName());
logger.log(Level.SEVERE, "File query failed", ex);
return IngestModule.ProcessResult.ERROR;
}
return IngestModule.ProcessResult.OK;
}
/**
* Invoked by Autopsy when an ingest job is completed, before the ingest
* module instance is discarded. The module should respond by doing things
* like releasing private resources, submitting final results, and posting a
* final ingest message.
* <p>
* Autopsy will generally use several instances of an ingest module for each
* ingest job it performs. Completing an ingest job entails processing a
* single data source (e.g., a disk image) and all of the files from the
* data source, including files extracted from archives and any unallocated
* space (made to look like a series of files). The data source is passed
* through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
* <p>
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that there will be no more than one module instance per
* thread. However, if the module instances must share resources, the
* modules are responsible for synchronizing access to the shared resources
* and doing reference counting as required to release those resources
* correctly. Also, more than one ingest job may be in progress at any given
* time. This must also be taken into consideration when sharing resources
* between module instances.
* <p>
* An ingest module that does not require initialization may extend the
* abstract IngestModuleAdapter class to get a default "do nothing"
* implementation of this method.
*/
@Override
public void shutDown(boolean ingestJobCancelled) {
// This method is thread-safe with per ingest job reference counting.
// This method is thread-safe with per ingest job reference counted
// management of shared data.
postFileCount(context.getJobId());
}
synchronized static void initFileCount(long ingestJobId) {
Integer moduleReferenceCount;
if (!moduleReferenceCountsForIngestJobs.containsKey(ingestJobId)) {
moduleReferenceCount = 1;
Long refCount = refCounter.incrementAndGet(ingestJobId);
if (refCount == 1) {
fileCountsForIngestJobs.put(ingestJobId, 0L);
} else {
moduleReferenceCount = moduleReferenceCountsForIngestJobs.get(ingestJobId);
++moduleReferenceCount;
}
moduleReferenceCountsForIngestJobs.put(ingestJobId, moduleReferenceCount);
}
synchronized static void addToFileCount(long ingestJobId, long countToAdd) {
@ -209,19 +144,16 @@ class SampleDataSourceIngestModule implements DataSourceIngestModule {
}
synchronized static void postFileCount(long ingestJobId) {
Integer moduleReferenceCount = moduleReferenceCountsForIngestJobs.remove(ingestJobId);
--moduleReferenceCount;
if (moduleReferenceCount == 0) {
Long refCount = refCounter.decrementAndGet(ingestJobId);
if (refCount == 0) {
Long filesCount = fileCountsForIngestJobs.remove(ingestJobId);
String msgText = String.format("Found %d files", filesCount);
IngestMessage message = IngestMessage.createMessage(
++messageCount,
IngestMessage.MessageType.DATA,
SampleIngestModuleFactory.getModuleName(),
msgText);
IngestServices.getInstance().postMessage(message);
} else {
moduleReferenceCountsForIngestJobs.put(ingestJobId, moduleReferenceCount);
}
}
}
}

106
Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java
View File

@ -37,8 +37,10 @@ import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
@ -50,57 +52,20 @@ import org.sleuthkit.datamodel.TskData;
/**
* Sample file ingest module that doesn't do much. Demonstrates per ingest job
* module settings, use of a subset of the available ingest services and
* thread-safe sharing of per ingest job resources.
* <p>
* IMPORTANT TIP: This sample data source ingest module directly implements
* FileIngestModule, which extends IngestModule. A practical alternative,
* recommended if you do not need to provide implementations of all of the
* IngestModule methods, is to extend the abstract class IngestModuleAdapter to
* get default "do nothing" implementations of the IngestModule methods.
* thread-safe sharing of per ingest job data.
*/
class SampleFileIngestModule implements FileIngestModule {
class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestModule {
private static final HashMap<Long, Integer> moduleReferenceCountsForIngestJobs = new HashMap<>();
private static final HashMap<Long, Long> artifactCountsForIngestJobs = new HashMap<>();
private static long messageCount = 0;
private static int attrId = -1;
private final boolean skipKnownFiles;
private IngestJobContext context = null;
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
SampleFileIngestModule(SampleModuleIngestJobSettings settings) {
this.skipKnownFiles = settings.skipKnownFiles();
}
/**
* Invoked by Autopsy to allow an ingest module instance to set up any
* internal data structures and acquire any private resources it will need
* during an ingest job.
* <p>
* Autopsy will generally use several instances of an ingest module for each
* ingest job it performs. Completing an ingest job entails processing a
* single data source (e.g., a disk image) and all of the files from the
* data source, including files extracted from archives and any unallocated
* space (made to look like a series of files). The data source is passed
* through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
* <p>
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that there will be no more than one module instance per
* thread. However, if the module instances must share resources, the
* modules are responsible for synchronizing access to the shared resources
* and doing reference counting as required to release those resources
* correctly. Also, more than one ingest job may be in progress at any given
* time. This must also be taken into consideration when sharing resources
* between module instances.
* <p>
* An ingest module that does not require initialization may extend the
* abstract IngestModuleAdapter class to get a default "do nothing"
* implementation of this method.
*
* @param context Provides data and services specific to the ingest job and
* the ingest pipeline of which the module is a part.
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
*/
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context;
@ -133,16 +98,11 @@ class SampleFileIngestModule implements FileIngestModule {
}
}
// This method is thread-safe with per ingest job reference counting.
// This method is thread-safe with per ingest job reference counted
// management of shared data.
initBlackboardPostCount(context.getJobId());
}
/**
* Processes a file.
*
* @param file The file.
* @return A result code indicating success or failure of the processing.
*/
@Override
public IngestModule.ProcessResult process(AbstractFile file) {
if (attrId != -1) {
@ -183,7 +143,8 @@ class SampleFileIngestModule implements FileIngestModule {
BlackboardArtifact art = file.getGenInfoArtifact();
art.addAttribute(attr);
// Thread-safe.
// This method is thread-safe with per ingest job reference counted
// management of shared data.
addToBlackboardPostCount(context.getJobId(), 1L);
// Fire an event to notify any listeners for blackboard postings.
@ -200,49 +161,18 @@ class SampleFileIngestModule implements FileIngestModule {
}
}
/**
* Invoked by Autopsy when an ingest job is completed, before the ingest
* module instance is discarded. The module should respond by doing things
* like releasing private resources, submitting final results, and posting a
* final ingest message.
* <p>
* Autopsy will generally use several instances of an ingest module for each
* ingest job it performs. Completing an ingest job entails processing a
* single data source (e.g., a disk image) and all of the files from the
* data source, including files extracted from archives and any unallocated
* space (made to look like a series of files). The data source is passed
* through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
* <p>
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that there will be no more than one module instance per
* thread. However, if the module instances must share resources, the
* modules are responsible for synchronizing access to the shared resources
* and doing reference counting as required to release those resources
* correctly. Also, more than one ingest job may be in progress at any given
* time. This must also be taken into consideration when sharing resources
* between module instances.
* <p>
* An ingest module that does not require initialization may extend the
* abstract IngestModuleAdapter class to get a default "do nothing"
* implementation of this method.
*/
@Override
public void shutDown(boolean ingestJobCancelled) {
// This method is thread-safe with per ingest job reference counting.
// This method is thread-safe with per ingest job reference counted
// management of shared data.
reportBlackboardPostCount(context.getJobId());
}
synchronized static void initBlackboardPostCount(long ingestJobId) {
Integer moduleReferenceCount;
if (!moduleReferenceCountsForIngestJobs.containsKey(ingestJobId)) {
moduleReferenceCount = 1;
Long refCount = refCounter.incrementAndGet(ingestJobId);
if (refCount == 1) {
artifactCountsForIngestJobs.put(ingestJobId, 0L);
} else {
moduleReferenceCount = moduleReferenceCountsForIngestJobs.get(ingestJobId);
++moduleReferenceCount;
}
moduleReferenceCountsForIngestJobs.put(ingestJobId, moduleReferenceCount);
}
synchronized static void addToBlackboardPostCount(long ingestJobId, long countToAdd) {
@ -252,19 +182,15 @@ class SampleFileIngestModule implements FileIngestModule {
}
synchronized static void reportBlackboardPostCount(long ingestJobId) {
Integer moduleReferenceCount = moduleReferenceCountsForIngestJobs.remove(ingestJobId);
--moduleReferenceCount;
if (moduleReferenceCount == 0) {
Long refCount = refCounter.decrementAndGet(ingestJobId);
if (refCount == 0) {
Long filesCount = artifactCountsForIngestJobs.remove(ingestJobId);
String msgText = String.format("Posted %d times to the blackboard", filesCount);
IngestMessage message = IngestMessage.createMessage(
++messageCount,
IngestMessage.MessageType.INFO,
SampleIngestModuleFactory.getModuleName(),
msgText);
IngestServices.getInstance().postMessage(message);
} else {
moduleReferenceCountsForIngestJobs.put(ingestJobId, moduleReferenceCount);
}
}
}
}

10
Core/src/org/sleuthkit/autopsy/examples/SampleIngestModuleFactory.java
View File

@ -55,10 +55,10 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
* create instances of a type of data source ingest module, a type of file
* ingest module, or both.
* <p>
* Autopsy will generally use the factory to several instances of each type of
* module for each ingest job it performs. Completing an ingest job entails
* processing a single data source (e.g., a disk image) and all of the files
* from the data source, including files extracted from archives and any
* Autopsy will generally use the factory to create several instances of each
* type of module for each ingest job it performs. Completing an ingest job
* entails processing a single data source (e.g., a disk image) and all of the
* files from the data source, including files extracted from archives and any
* unallocated space (made to look like a series of files). The data source is
* passed through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
@ -87,7 +87,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
* extend the abstract class IngestModuleFactoryAdapter to get default
* implementations of most of the IngestModuleFactory methods.
*/
// @ServiceProvider(service = IngestModuleFactory.class)
@ServiceProvider(service = IngestModuleFactory.class) // Sample is discarded at runtime
public class SampleIngestModuleFactory implements IngestModuleFactory {
private static final String VERSION_NUMBER = "1.0.0";

13
Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties
View File

@ -16,9 +16,11 @@ IngestMessagePanel.totalMessagesNameLabel.text=Total:
IngestMessagePanel.totalMessagesNameVal.text=-
IngestMessagePanel.totalUniqueMessagesNameLabel.text=Unique:
IngestMessagePanel.totalUniqueMessagesNameVal.text=-
IngestJob.progress.dataSourceIngest.displayName=Data Source Ingest of {0}
IngestJob.progress.fileIngest.displayName=File Ingest of {0}
IngestJob.progress.cancelling={0} (Cancelling...)
IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=Processes unallocated space, such as deleted files. Produces more complete results, but it may take longer to process on large images.
IngestJobConfigurationPanel.processUnallocCheckbox.text=Process Unallocated Space
IngestJobConfigurationPanel.advancedButton.text=Advanced
IngestJob.toString.text=ScheduledTask'{'input\={0}, modules\={1}'}'
IngestJobLauncher.modName.tbirdParser.text=Thunderbird Parser
IngestJobLauncher.modName.mboxParser.text=MBox Parser
@ -46,14 +48,8 @@ IngestManager.toHtmlStr.totalErrs.text=Total errors\: {0}
IngestManager.toHtmlStr.module.text=Module
IngestManager.toHtmlStr.time.text=Time
IngestManager.toHtmlStr.errors.text=Errors
IngestManager.FileTaskWorker.displayName=File Ingest
IngestManager.FileTaskWorker.process.cancelling={0} (Cancelling...)
IngestManager.EnqueueWorker.displayName.text=Queueing Ingest
IngestManager.EnqueueWorker.process.cancelling={0} (Cancelling...)
IngestManager.DataSourceTaskWorker.progress.pending={0} (Pending...)
IngestManager.DataSourceTaskWorker.progress.cancelling={0} (Cancelling...)
IngestManager.datatSourceIngest.progress.text=DataSource Ingest {0}
IngestManager.fileIngest.progress.text=File Ingest {0}
IngestJob.DataSourceIngestPipeline.displayName.text={0} processing {1}
IngestMessage.toString.type.text=type\: {0}
IngestMessage.toString.source.text=\ source\: {0}
@ -101,3 +97,6 @@ IngestScheduler.remove.exception.notSupported.msg=Not supported.
IngestScheduler.DataSourceScheduler.exception.next.msg=There is no data source tasks in the queue, check hasNext()
IngestScheduler.DataSourceScheduler.exception.remove.msg=Removing of scheduled data source ingest tasks is not supported.
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\: {0}
Label1
IngestJobConfigurationPanel.advancedButton.text=Advanced
IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced

6
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java
View File

@ -22,16 +22,18 @@ import org.sleuthkit.datamodel.Content;
/**
* Interface that must be implemented by all data source ingest modules.
* See description of IngestModule for more details on interface behavior.
*/
public interface DataSourceIngestModule extends IngestModule {
/**
* Processes a data source.
* Processes a data source. Called once between calls to startUp()
* and shutDown().
*
* @param dataSource The data source to process.
* @param statusHelper A status helper to be used to report progress and
* detect ingest job cancellation.
* @return A result code indicating success or failure of the processing.
*/
ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper);
ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper);
}

44
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java → Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java
View File

@ -18,35 +18,17 @@
*/
package org.sleuthkit.autopsy.ingest;
import javax.swing.SwingWorker;
import org.netbeans.api.progress.ProgressHandle;
import org.sleuthkit.datamodel.Content;
/**
* Used by data source ingest modules to report progress and detect data source
* ingest job cancellation.
* Used by data source ingest modules to report progress.
*/
public class DataSourceIngestModuleStatusHelper {
public class DataSourceIngestModuleProgress {
private final SwingWorker worker;
private final ProgressHandle progress;
private final Content dataSource;
private final IngestJob ingestJob;
private final String moduleDisplayName;
DataSourceIngestModuleStatusHelper(SwingWorker worker, ProgressHandle progress, Content dataSource) {
this.worker = worker;
this.progress = progress;
this.dataSource = dataSource;
}
/**
* Checks for ingest job cancellation. This should be polled by the module
* in its process() method. If the ingest task is canceled, the module
* should return from its process() method as quickly as possible.
*
* @return True if the task has been canceled, false otherwise.
*/
public boolean isIngestJobCancelled() {
return worker.isCancelled();
DataSourceIngestModuleProgress(IngestJob ingestJob, String moduleDisplayName) {
this.ingestJob = ingestJob;
this.moduleDisplayName = moduleDisplayName;
}
/**
@ -58,9 +40,7 @@ public class DataSourceIngestModuleStatusHelper {
* data source.
*/
public void switchToDeterminate(int workUnits) {
if (progress != null) {
progress.switchToDeterminate(workUnits);
}
ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits);
}
/**
@ -68,9 +48,7 @@ public class DataSourceIngestModuleStatusHelper {
* the total work units to process the data source is unknown.
*/
public void switchToIndeterminate() {
if (progress != null) {
progress.switchToIndeterminate();
}
ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate();
}
/**
@ -80,8 +58,6 @@ public class DataSourceIngestModuleStatusHelper {
* @param workUnits Number of work units performed so far by the module.
*/
public void progress(int workUnits) {
if (progress != null) {
progress.progress(dataSource.getName(), workUnits);
}
ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits);
}
}

138
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java Executable file
View File

@ -0,0 +1,138 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sleuthkit.datamodel.Content;
/**
* A data source ingest pipeline composed of a sequence of data source ingest
* modules constructed from ingest module templates.
*/
final class DataSourceIngestPipeline {
private final IngestJob job;
private final List<IngestModuleTemplate> moduleTemplates;
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
DataSourceIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) {
this.job = task;
this.moduleTemplates = moduleTemplates;
}
List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
// Create an ingest module instance from each ingest module template
// that has an ingest module factory capable of making data source
// ingest modules. Map the module class names to the module instance
// to allow the modules to be put in the sequence indicated by the
// ingest pipelines configuration.
Map<String, DataSourceIngestModuleDecorator> modulesByClass = new HashMap<>();
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try {
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
}
// Establish the module sequence of the core ingest modules
// indicated by the ingest pipeline configuration, adding any
// additional modules found in the global lookup to the end of the
// pipeline in arbitrary order.
List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getDataSourceIngestPipelineConfig();
for (String moduleClassName : pipelineConfig) {
if (modulesByClass.containsKey(moduleClassName)) {
modules.add(modulesByClass.remove(moduleClassName));
}
}
for (DataSourceIngestModuleDecorator module : modulesByClass.values()) {
modules.add(module);
}
return errors;
}
List<IngestModuleError> process() {
List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) {
try {
module.process(job.getDataSource(), new DataSourceIngestModuleProgress(job, module.getDisplayName()));
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (job.isCancelled()) {
break;
}
}
return errors;
}
List<IngestModuleError> shutDown(boolean ingestJobCancelled) {
List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) {
try {
module.shutDown(ingestJobCancelled);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
return errors;
}
private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule {
private final DataSourceIngestModule module;
private final String displayName;
DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) {
this.module = module;
this.displayName = displayName;
}
String getClassName() {
return module.getClass().getCanonicalName();
}
String getDisplayName() {
return displayName;
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
module.startUp(context);
}
@Override
public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
return module.process(dataSource, statusHelper);
}
@Override
public void shutDown(boolean ingestJobWasCancelled) {
module.shutDown(ingestJobWasCancelled);
}
}
}

6
Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java
View File

@ -22,13 +22,15 @@ import org.sleuthkit.datamodel.AbstractFile;
/**
* Interface that must be implemented by all file ingest modules.
* See description of IngestModule for more details on interface behavior.
*/
public interface FileIngestModule extends IngestModule {
/**
* Processes a file.
* Processes a file. Called between calls to startUp() and shutDown().
* Will be called for each file in a data source.
*
* @param file The file.
* @param file The file to analyze.
* @return A result code indicating success or failure of the processing.
*/
ProcessResult process(AbstractFile file);

142
Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java Executable file
View File

@ -0,0 +1,142 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sleuthkit.datamodel.AbstractFile;
/**
* A file ingest pipeline composed of a sequence of file ingest modules
* constructed from ingest module templates.
*/
final class FileIngestPipeline {
private final IngestJob job;
private final List<IngestModuleTemplate> moduleTemplates;
private List<FileIngestModuleDecorator> modules = new ArrayList<>();
FileIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) {
this.job = task;
this.moduleTemplates = moduleTemplates;
}
List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
// Create an ingest module instance from each ingest module template
// that has an ingest module factory capable of making data source
// ingest modules. Map the module class names to the module instance
// to allow the modules to be put in the sequence indicated by the
// ingest pipelines configuration.
Map<String, FileIngestModuleDecorator> modulesByClass = new HashMap<>();
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isFileIngestModuleTemplate()) {
FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try {
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
}
// Establish the module sequence of the core ingest modules
// indicated by the ingest pipeline configuration, adding any
// additional modules found in the global lookup to the end of the
// pipeline in arbitrary order.
List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getFileIngestPipelineConfig();
for (String moduleClassName : pipelineConfig) {
if (modulesByClass.containsKey(moduleClassName)) {
modules.add(modulesByClass.remove(moduleClassName));
}
}
for (FileIngestModuleDecorator module : modulesByClass.values()) {
modules.add(module);
}
return errors;
}
List<IngestModuleError> process(AbstractFile file) {
List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) {
try {
module.process(file);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (job.isCancelled()) {
break;
}
}
file.close();
if (!job.isCancelled()) {
IngestManager.fireFileIngestDone(file.getId());
}
return errors;
}
List<IngestModuleError> shutDown(boolean ingestJobCancelled) {
List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) {
try {
module.shutDown(ingestJobCancelled);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
return errors;
}
private static class FileIngestModuleDecorator implements FileIngestModule {
private final FileIngestModule module;
private final String displayName;
FileIngestModuleDecorator(FileIngestModule module, String displayName) {
this.module = module;
this.displayName = displayName;
}
String getClassName() {
return module.getClass().getCanonicalName();
}
String getDisplayName() {
return displayName;
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
module.startUp(context);
}
@Override
public IngestModule.ProcessResult process(AbstractFile file) {
return module.process(file);
}
@Override
public void shutDown(boolean ingestJobWasCancelled) {
module.shutDown(ingestJobWasCancelled);
}
}
}

354
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -21,13 +21,10 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import javax.swing.SwingWorker;
import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
/**
@ -42,9 +39,14 @@ final class IngestJob {
private final boolean processUnallocatedSpace;
private final HashMap<Long, FileIngestPipeline> fileIngestPipelines = new HashMap<>();
private final HashMap<Long, DataSourceIngestPipeline> dataSourceIngestPipelines = new HashMap<>();
private final IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler();
private FileIngestPipeline initialFileIngestPipeline = null;
private DataSourceIngestPipeline initialDataSourceIngestPipeline = null;
private boolean cancelled;
private ProgressHandle dataSourceTaskProgress;
private ProgressHandle fileTasksProgress;
int totalEnqueuedFiles = 0;
private int processedFiles = 0;
private volatile boolean cancelled;
IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) {
this.id = id;
@ -66,15 +68,53 @@ final class IngestJob {
return processUnallocatedSpace;
}
synchronized void cancel() {
cancelled = true;
}
synchronized boolean isCancelled() {
return cancelled;
}
synchronized List<IngestModuleError> startUpIngestPipelines() {
startDataSourceIngestProgressBar();
startFileIngestProgressBar();
return startUpInitialIngestPipelines();
}
private void startDataSourceIngestProgressBar() {
final String displayName = NbBundle
.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", this.dataSource.getName());
dataSourceTaskProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
if (dataSourceTaskProgress != null) {
dataSourceTaskProgress.setDisplayName(NbBundle.getMessage(this.getClass(),
"IngestJob.progress.cancelling",
displayName));
}
IngestManager.getInstance().cancelIngestJobs();
return true;
}
});
dataSourceTaskProgress.start();
dataSourceTaskProgress.switchToIndeterminate();
}
private void startFileIngestProgressBar() {
final String displayName = NbBundle
.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName());
fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
if (fileTasksProgress != null) {
fileTasksProgress.setDisplayName(
NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
displayName));
}
IngestManager.getInstance().cancelIngestJobs();
return true;
}
});
fileTasksProgress.start();
fileTasksProgress.switchToIndeterminate();
totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
}
private List<IngestModuleError> startUpInitialIngestPipelines() {
// Create a per thread instance of each pipeline type right now to make
// (reasonably) sure that the ingest modules can be started.
initialDataSourceIngestPipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates);
@ -119,271 +159,69 @@ final class IngestJob {
synchronized List<IngestModuleError> releaseIngestPipelinesForThread(long threadId) {
List<IngestModuleError> errors = new ArrayList<>();
DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId);
if (dataSourceIngestPipeline != null) {
errors.addAll(dataSourceIngestPipeline.shutDown(cancelled));
dataSourceIngestPipelines.remove(threadId);
}
if (initialDataSourceIngestPipeline == null && dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) {
dataSourceTaskProgress.finish();
dataSourceTaskProgress = null;
}
this.dataSourceIngestPipelines.remove(threadId);
FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId);
if (fileIngestPipeline != null) {
errors.addAll(fileIngestPipeline.shutDown(cancelled));
fileIngestPipelines.remove(threadId);
}
if (initialFileIngestPipeline == null && fileIngestPipelines.isEmpty() && fileTasksProgress != null) {
fileTasksProgress.finish();
fileTasksProgress = null;
}
this.fileIngestPipelines.remove(threadId);
return errors;
}
synchronized boolean areIngestPipelinesShutDown() {
return (dataSourceIngestPipelines.isEmpty() && fileIngestPipelines.isEmpty());
return (initialDataSourceIngestPipeline == null
&& dataSourceIngestPipelines.isEmpty()
&& initialFileIngestPipeline == null
&& fileIngestPipelines.isEmpty());
}
/**
* A data source ingest pipeline composed of a sequence of data source ingest
* modules constructed from ingest module templates.
*/
static final class DataSourceIngestPipeline {
private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName());
private final IngestJob task;
private final List<IngestModuleTemplate> moduleTemplates;
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
private DataSourceIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) {
this.task = task;
this.moduleTemplates = moduleTemplates;
}
private List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
// Create an ingest module instance from each ingest module template
// that has an ingest module factory capable of making data source
// ingest modules. Map the module class names to the module instance
// to allow the modules to be put in the sequence indicated by the
// ingest pipelines configuration.
Map<String, DataSourceIngestModuleDecorator> modulesByClass = new HashMap<>();
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(task);
try {
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), module.getDisplayName());
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
}
// Establish the module sequence of the core ingest modules
// indicated by the ingest pipeline configuration, adding any
// additional modules found in the global lookup to the end of the
// pipeline in arbitrary order.
List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getDataSourceIngestPipelineConfig();
for (String moduleClassName : pipelineConfig) {
if (modulesByClass.containsKey(moduleClassName)) {
modules.add(modulesByClass.remove(moduleClassName));
}
}
for (DataSourceIngestModuleDecorator module : modulesByClass.values()) {
modules.add(module);
}
return errors;
}
List<IngestModuleError> process(SwingWorker worker, ProgressHandle progress) {
List<IngestModuleError> errors = new ArrayList<>();
Content dataSource = this.task.getDataSource();
logger.log(Level.INFO, "Processing data source {0}", dataSource.getName());
for (DataSourceIngestModuleDecorator module : this.modules) {
try {
String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.DataSourceIngestPipeline.displayName.text", module.getDisplayName(), dataSource.getName());
progress.setDisplayName(displayName);
module.process(dataSource, new DataSourceIngestModuleStatusHelper(worker, progress, dataSource));
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (task.isCancelled()) {
break;
}
}
return errors;
}
private List<IngestModuleError> shutDown(boolean ingestJobCancelled) {
List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) {
try {
module.shutDown(ingestJobCancelled);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
} finally {
IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName());
}
}
return errors;
}
private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule {
private final DataSourceIngestModule module;
private final String displayName;
DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) {
this.module = module;
this.displayName = displayName;
}
String getClassName() {
return module.getClass().getCanonicalName();
}
String getDisplayName() {
return displayName;
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
module.startUp(context);
}
@Override
public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) {
return module.process(dataSource, statusHelper);
}
@Override
public void shutDown(boolean ingestJobWasCancelled) {
module.shutDown(ingestJobWasCancelled);
}
}
synchronized ProgressHandle getDataSourceTaskProgressBar() {
return this.dataSourceTaskProgress;
}
/**
* A file ingest pipeline composed of a sequence of file ingest modules
* constructed from ingest module templates.
*/
static final class FileIngestPipeline {
private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName());
private final IngestJob task;
private final List<IngestModuleTemplate> moduleTemplates;
private List<FileIngestModuleDecorator> modules = new ArrayList<>();
private FileIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) {
this.task = task;
this.moduleTemplates = moduleTemplates;
synchronized void updateFileTasksProgressBar(String currentFileName) {
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
totalEnqueuedFiles = newTotalEnqueuedFiles + 1;
fileTasksProgress.switchToIndeterminate();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
}
if (processedFiles < totalEnqueuedFiles) {
++processedFiles;
}
private List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
// Create an ingest module instance from each ingest module template
// that has an ingest module factory capable of making data source
// ingest modules. Map the module class names to the module instance
// to allow the modules to be put in the sequence indicated by the
// ingest pipelines configuration.
Map<String, FileIngestModuleDecorator> modulesByClass = new HashMap<>();
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isFileIngestModuleTemplate()) {
FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); // RJCTODO: Move into try block for bpth impls
IngestJobContext context = new IngestJobContext(task);
try {
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), template.getModuleName());
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
}
// Establish the module sequence of the core ingest modules
// indicated by the ingest pipeline configuration, adding any
// additional modules found in the global lookup to the end of the
// pipeline in arbitrary order.
List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getFileIngestPipelineConfig();
for (String moduleClassName : pipelineConfig) {
if (modulesByClass.containsKey(moduleClassName)) {
modules.add(modulesByClass.remove(moduleClassName));
}
}
for (FileIngestModuleDecorator module : modulesByClass.values()) {
modules.add(module);
}
return errors;
fileTasksProgress.progress(currentFileName, processedFiles);
}
synchronized void cancel() {
if (initialDataSourceIngestPipeline != null) {
initialDataSourceIngestPipeline.shutDown(true);
initialDataSourceIngestPipeline = null;
}
List<IngestModuleError> process(AbstractFile file) {
List<IngestModuleError> errors = new ArrayList<>();
Content dataSource = this.task.getDataSource();
logger.log(Level.INFO, String.format("Processing {0} from {1}", file.getName(), dataSource.getName()));
for (FileIngestModuleDecorator module : this.modules) {
try {
module.process(file);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (task.isCancelled()) {
break;
}
}
file.close();
IngestManager.fireFileDone(file.getId());
return errors;
if (initialFileIngestPipeline != null) {
initialFileIngestPipeline.shutDown(true);
initialFileIngestPipeline = null;
}
cancelled = true;
}
private List<IngestModuleError> shutDown(boolean ingestJobCancelled) {
List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) {
try {
module.shutDown(ingestJobCancelled);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
} finally {
IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName());
}
}
return errors;
}
private static class FileIngestModuleDecorator implements FileIngestModule {
private final FileIngestModule module;
private final String displayName;
FileIngestModuleDecorator(FileIngestModule module, String displayName) {
this.module = module;
this.displayName = displayName;
}
String getClassName() {
return module.getClass().getCanonicalName();
}
String getDisplayName() {
return displayName;
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
module.startUp(context);
}
@Override
public IngestModule.ProcessResult process(AbstractFile file) {
return module.process(file);
}
@Override
public void shutDown(boolean ingestJobWasCancelled) {
module.shutDown(ingestJobWasCancelled);
}
}
}
boolean isCancelled() {
return cancelled;
}
}

83
Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form
View File

@ -13,7 +13,7 @@
<Dimension value="[522, 257]"/>
</Property>
<Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[575, 300]"/>
<Dimension value="[575, 400]"/>
</Property>
</Properties>
<AuxValues>
@ -48,9 +48,9 @@
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jPanel1" pref="278" max="32767" attributes="0"/>
<Component id="jPanel1" pref="342" max="32767" attributes="0"/>
<Group type="102" attributes="0">
<Component id="modulesScrollPane" pref="233" max="32767" attributes="0"/>
<Component id="modulesScrollPane" max="32767" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="processUnallocPanel" max="-2" attributes="0"/>
</Group>
@ -108,24 +108,40 @@
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jScrollPane1" alignment="0" pref="326" max="32767" attributes="0"/>
<Component id="jSeparator2" alignment="1" max="32767" attributes="0"/>
<Group type="102" alignment="1" attributes="0">
<EmptySpace max="32767" attributes="0"/>
<Component id="advancedButton" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Group type="102" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jScrollPane1" pref="316" max="32767" attributes="0"/>
<Group type="102" attributes="0">
<Component id="scrollpane" max="32767" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="advancedButton" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="14" max="-2" attributes="0"/>
</Group>
</Group>
</Group>
<Component id="jSeparator2" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="1" attributes="0">
<Component id="jScrollPane1" max="32767" attributes="0"/>
<Component id="jScrollPane1" pref="242" max="32767" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="22" max="-2" attributes="0"/>
<Component id="scrollpane" min="-2" pref="65" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="jSeparator2" min="-2" pref="10" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace type="separate" max="-2" attributes="0"/>
<Component id="advancedButton" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace max="-2" attributes="0"/>
<Component id="jSeparator2" min="-2" pref="10" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="0" max="-2" attributes="0"/>
<Component id="advancedButton" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -136,6 +152,9 @@
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/ingest/Bundle.properties" key="IngestJobConfigurationPanel.advancedButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="actionCommand" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/ingest/Bundle.properties" key="IngestJobConfigurationPanel.advancedButton.actionCommand" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="enabled" type="boolean" value="false"/>
</Properties>
<Events>
@ -164,6 +183,40 @@
</Container>
</SubComponents>
</Container>
<Container class="javax.swing.JScrollPane" name="scrollpane">
<Properties>
<Property name="border" type="javax.swing.border.Border" editor="org.netbeans.modules.form.editors2.BorderEditor">
<Border info="null"/>
</Property>
<Property name="horizontalScrollBarPolicy" type="int" value="31"/>
<Property name="verticalScrollBarPolicy" type="int" value="21"/>
</Properties>
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
<SubComponents>
<Component class="javax.swing.JTextArea" name="descriptionLabel">
<Properties>
<Property name="editable" type="boolean" value="false"/>
<Property name="background" type="java.awt.Color" editor="org.netbeans.beaninfo.editors.ColorEditor">
<Color blue="f0" green="f0" red="f0" type="rgb"/>
</Property>
<Property name="columns" type="int" value="20"/>
<Property name="font" type="java.awt.Font" editor="org.netbeans.beaninfo.editors.FontEditor">
<Font name="Tahoma" size="11" style="0"/>
</Property>
<Property name="lineWrap" type="boolean" value="true"/>
<Property name="rows" type="int" value="5"/>
<Property name="wrapStyleWord" type="boolean" value="true"/>
<Property name="border" type="javax.swing.border.Border" editor="org.netbeans.modules.form.editors2.BorderEditor">
<Border info="null"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Container>
</SubComponents>
</Container>
<Container class="javax.swing.JPanel" name="processUnallocPanel">
@ -183,7 +236,7 @@
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="processUnallocCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="60" max="32767" attributes="0"/>
<EmptySpace pref="108" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>

67
Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java
View File

@ -107,6 +107,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
simplePanel.revalidate();
simplePanel.repaint();
advancedButton.setEnabled(null != selectedModule.getGlobalSettingsPanel());
descriptionLabel.setText(selectedModule.getDescription());
}
}
});
@ -131,12 +132,14 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
jSeparator2 = new javax.swing.JSeparator();
jScrollPane1 = new javax.swing.JScrollPane();
simplePanel = new javax.swing.JPanel();
scrollpane = new javax.swing.JScrollPane();
descriptionLabel = new javax.swing.JTextArea();
processUnallocPanel = new javax.swing.JPanel();
processUnallocCheckbox = new javax.swing.JCheckBox();
setMaximumSize(new java.awt.Dimension(5750, 3000));
setMinimumSize(new java.awt.Dimension(522, 257));
setPreferredSize(new java.awt.Dimension(575, 300));
setPreferredSize(new java.awt.Dimension(575, 400));
modulesScrollPane.setBorder(javax.swing.BorderFactory.createLineBorder(new java.awt.Color(160, 160, 160)));
modulesScrollPane.setPreferredSize(new java.awt.Dimension(160, 160));
@ -158,6 +161,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
jPanel1.setPreferredSize(new java.awt.Dimension(338, 257));
advancedButton.setText(org.openide.util.NbBundle.getMessage(IngestJobConfigurationPanel.class, "IngestJobConfigurationPanel.advancedButton.text")); // NOI18N
advancedButton.setActionCommand(org.openide.util.NbBundle.getMessage(IngestJobConfigurationPanel.class, "IngestJobConfigurationPanel.advancedButton.actionCommand")); // NOI18N
advancedButton.setEnabled(false);
advancedButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
@ -171,25 +175,49 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
simplePanel.setLayout(new javax.swing.BoxLayout(simplePanel, javax.swing.BoxLayout.PAGE_AXIS));
jScrollPane1.setViewportView(simplePanel);
scrollpane.setBorder(null);
scrollpane.setHorizontalScrollBarPolicy(javax.swing.ScrollPaneConstants.HORIZONTAL_SCROLLBAR_NEVER);
scrollpane.setVerticalScrollBarPolicy(javax.swing.ScrollPaneConstants.VERTICAL_SCROLLBAR_NEVER);
descriptionLabel.setEditable(false);
descriptionLabel.setBackground(new java.awt.Color(240, 240, 240));
descriptionLabel.setColumns(20);
descriptionLabel.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N
descriptionLabel.setLineWrap(true);
descriptionLabel.setRows(5);
descriptionLabel.setWrapStyleWord(true);
descriptionLabel.setBorder(null);
scrollpane.setViewportView(descriptionLabel);
javax.swing.GroupLayout jPanel1Layout = new javax.swing.GroupLayout(jPanel1);
jPanel1.setLayout(jPanel1Layout);
jPanel1Layout.setHorizontalGroup(
jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 326, Short.MAX_VALUE)
.addComponent(jSeparator2, javax.swing.GroupLayout.Alignment.TRAILING)
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup()
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(advancedButton)
.addContainerGap())
.addGroup(jPanel1Layout.createSequentialGroup()
.addContainerGap()
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 316, Short.MAX_VALUE)
.addGroup(jPanel1Layout.createSequentialGroup()
.addComponent(scrollpane)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(advancedButton)
.addGap(14, 14, 14))))
.addComponent(jSeparator2)
);
jPanel1Layout.setVerticalGroup(
jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup()
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(jSeparator2, javax.swing.GroupLayout.PREFERRED_SIZE, 10, javax.swing.GroupLayout.PREFERRED_SIZE)
.addGap(0, 0, 0)
.addComponent(advancedButton)
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 242, Short.MAX_VALUE)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(jPanel1Layout.createSequentialGroup()
.addGap(22, 22, 22)
.addComponent(scrollpane, javax.swing.GroupLayout.PREFERRED_SIZE, 65, javax.swing.GroupLayout.PREFERRED_SIZE))
.addGroup(jPanel1Layout.createSequentialGroup()
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(jSeparator2, javax.swing.GroupLayout.PREFERRED_SIZE, 10, javax.swing.GroupLayout.PREFERRED_SIZE))
.addGroup(jPanel1Layout.createSequentialGroup()
.addGap(18, 18, 18)
.addComponent(advancedButton)))
.addContainerGap())
);
@ -210,7 +238,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
.addGroup(processUnallocPanelLayout.createSequentialGroup()
.addContainerGap()
.addComponent(processUnallocCheckbox)
.addContainerGap(60, Short.MAX_VALUE))
.addContainerGap(108, Short.MAX_VALUE))
);
processUnallocPanelLayout.setVerticalGroup(
processUnallocPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@ -238,15 +266,19 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jPanel1, javax.swing.GroupLayout.DEFAULT_SIZE, 278, Short.MAX_VALUE)
.addComponent(jPanel1, javax.swing.GroupLayout.DEFAULT_SIZE, 342, Short.MAX_VALUE)
.addGroup(layout.createSequentialGroup()
.addComponent(modulesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 233, Short.MAX_VALUE)
.addComponent(modulesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(processUnallocPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)))
.addContainerGap())
);
}// </editor-fold>//GEN-END:initComponents
private void processUnallocCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_processUnallocCheckboxActionPerformed
processUnallocatedSpace = processUnallocCheckbox.isSelected();
}//GEN-LAST:event_processUnallocCheckboxActionPerformed
private void advancedButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_advancedButtonActionPerformed
final AdvancedConfigurationDialog dialog = new AdvancedConfigurationDialog();
@ -270,11 +302,9 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
dialog.display(selectedModule.getGlobalSettingsPanel());
}//GEN-LAST:event_advancedButtonActionPerformed
private void processUnallocCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_processUnallocCheckboxActionPerformed
processUnallocatedSpace = processUnallocCheckbox.isSelected();
}//GEN-LAST:event_processUnallocCheckboxActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton advancedButton;
private javax.swing.JTextArea descriptionLabel;
private javax.swing.JPanel jPanel1;
private javax.swing.JScrollPane jScrollPane1;
private javax.swing.JSeparator jSeparator2;
@ -282,6 +312,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel {
private javax.swing.JTable modulesTable;
private javax.swing.JCheckBox processUnallocCheckbox;
private javax.swing.JPanel processUnallocPanel;
private javax.swing.JScrollPane scrollpane;
private javax.swing.JPanel simplePanel;
private javax.swing.ButtonGroup timeGroup;
// End of variables declaration//GEN-END:variables

2
Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java
View File

@ -60,7 +60,7 @@ public final class IngestJobContext {
*/
public void addFiles(List<AbstractFile> files) {
for (AbstractFile file : files) {
IngestManager.getInstance().scheduleFile(ingestJob.getId(), file);
IngestManager.getInstance().addFileToIngestJob(ingestJob.getId(), file);
}
}
}

4
Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java
View File

@ -92,7 +92,7 @@ public final class IngestJobLauncher {
if (enabledModuleNames.contains(moduleName)) {
moduleTemplate.setEnabled(true);
} else if (disabledModuleNames.contains(moduleName)) {
moduleTemplate.setEnabled(true);
moduleTemplate.setEnabled(false);
} else {
// The module factory was loaded, but the module name does not
// appear in the enabled/disabled module settings. Treat the
@ -232,7 +232,7 @@ public final class IngestJobLauncher {
}
if ((!enabledModuleTemplates.isEmpty()) && (dataSources != null) && (!dataSources.isEmpty())) {
IngestManager.getInstance().scheduleDataSourceTasks(dataSources, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace());
IngestManager.getInstance().startIngestJobs(dataSources, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace());
}
}
}

751
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -24,65 +24,143 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import javax.swing.SwingWorker;
import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable;
import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import java.util.prefs.Preferences;
import javax.swing.SwingWorker;
/**
* Manages the execution of ingest jobs.
*/
public class IngestManager {
private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads";
private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1;
private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4;
private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2;
private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class);
private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class);
private static IngestManager instance;
private final IngestScheduler scheduler;
private final IngestScheduler scheduler = IngestScheduler.getInstance();
private final IngestMonitor ingestMonitor = new IngestMonitor();
private final HashMap<Long, IngestJob> ingestJobs = new HashMap<>();
private TaskSchedulingWorker taskSchedulingWorker;
private FileTaskWorker fileTaskWorker;
private DataSourceTaskWorker dataSourceTaskWorker;
private long nextDataSourceTaskId = 0;
private long nextThreadId = 0;
private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor();
private final ExecutorService dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor();
private final ExecutorService fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS);
private final HashMap<Long, IngestJob> ingestJobs = new HashMap<>(); // Maps job ids to jobs
private final HashMap<Long, Future<?>> ingestTasks = new HashMap<>(); // Maps task ids to task cancellation handles
private AtomicLong ingestJobId = new AtomicLong(0L);
private AtomicLong ingestTaskId = new AtomicLong(0L);
private volatile IngestUI ingestMessageBox;
/**
* Gets the IngestManager singleton, creating it if necessary.
*
* @returns The IngestManager singleton.
*/
public synchronized static IngestManager getInstance() {
if (instance == null) {
instance = new IngestManager();
}
return instance;
}
private IngestManager() {
}
/**
* Finds the top component for the ingest messages in box. Called by the
* custom installer for this package once the window system is initialized.
*/
void initIngestMessageInbox() {
if (this.ingestMessageBox == null) {
this.ingestMessageBox = IngestMessageTopComponent.findInstance();
}
}
public synchronized static int getNumberOfFileIngestThreads() {
return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS);
}
public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) {
if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS
|| numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) {
numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS;
}
userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads);
}
synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
if (!isIngestRunning() && ingestMessageBox != null) {
ingestMessageBox.clearMessages();
}
long taskId = ingestTaskId.incrementAndGet();
Future<?> task = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
ingestTasks.put(taskId, task);
if (ingestMessageBox != null) {
ingestMessageBox.restoreMessages();
}
}
/**
* Test if any ingest jobs are in progress.
*
* @return True if any ingest jobs are in progress, false otherwise
*/
public boolean isIngestRunning() {
return (ingestJobs.isEmpty() == false);
}
synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) {
IngestJob job = ingestJobs.get(ingestJobId);
if (job != null) {
scheduler.getFileScheduler().scheduleFile(job, file);
}
}
void cancelIngestJobs() {
new IngestCancellationWorker().execute();
}
/**
* Ingest events.
*/
public enum IngestEvent {
// RJCTODO: Update comments
/**
* Event sent when an ingest module has been started. Second argument of
* the property change is a string form of the module name and the third
* argument is null.
* Property change event fired when an ingest job is started. The ingest
* job id is in old value field of the PropertyChangeEvent object.
*/
STARTED,
INGEST_JOB_STARTED,
/**
* Event sent when an ingest module has completed processing by its own
* means. Second argument of the property change is a string form of the
* module name and the third argument is null.
*
* This event is generally used by listeners to perform a final data
* view refresh (listeners need to query all data from the blackboard).
* Property change event fired when an ingest job is completed. The
* ingest job id is in old value field of the PropertyChangeEvent
* object.
*/
COMPLETED,
INGEST_JOB_COMPLETED,
/**
* Event sent when an ingest module has stopped processing, and likely
* not all data has been processed. Second argument of the property
* change is a string form of the module name and third argument is
* null.
* Property change event fired when an ingest job is canceled. The
* ingest job id is in old value field of the PropertyChangeEvent
* object.
*/
STOPPED,
INGEST_JOB_CANCELLED,
/**
* Event sent when ingest module posts new data to blackboard or
* Event sent when an ingest module posts new data to blackboard or
* somewhere else. Second argument of the property change fired contains
* ModuleDataEvent object and third argument is null. The object can
* contain encapsulated new data created by the module. Listener can
@ -102,56 +180,22 @@ public class IngestManager {
FILE_DONE,
};
private IngestManager() {
this.scheduler = IngestScheduler.getInstance();
}
/**
* Returns reference to singleton instance.
*
* @returns Instance of class.
*/
synchronized public static IngestManager getInstance() {
if (instance == null) {
instance = new IngestManager();
}
return instance;
}
/**
* called by Installer in AWT thread once the Window System is ready
*/
void initIngestMessageInbox() {
if (this.ingestMessageBox == null) {
this.ingestMessageBox = IngestMessageTopComponent.findInstance();
}
}
synchronized private long getNextDataSourceTaskId() {
return ++this.nextDataSourceTaskId;
}
synchronized private long getNextThreadId() {
return ++this.nextThreadId;
}
/**
* Add property change listener to listen to ingest events as defined in
* IngestModuleEvent.
* Add property change listener to listen to ingest events.
*
* @param listener PropertyChangeListener to register
*/
public static synchronized void addPropertyChangeListener(final PropertyChangeListener listener) {
public static void addPropertyChangeListener(final PropertyChangeListener listener) {
pcs.addPropertyChangeListener(listener);
}
public static synchronized void removePropertyChangeListener(final PropertyChangeListener listener) {
public static void removePropertyChangeListener(final PropertyChangeListener listener) {
pcs.removePropertyChangeListener(listener);
}
static synchronized void fireModuleEvent(String eventType, String moduleName) {
static void fireIngestJobEvent(String eventType, long jobId) {
try {
pcs.firePropertyChange(eventType, moduleName, null);
pcs.firePropertyChange(eventType, jobId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e);
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
@ -163,11 +207,11 @@ public class IngestManager {
/**
* Fire event when file is done with a pipeline run
*
* @param objId ID of file that is done
* @param fileId ID of file that is done
*/
static synchronized void fireFileDone(long objId) {
static void fireFileIngestDone(long fileId) {
try {
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), objId, null);
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e);
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
@ -182,7 +226,7 @@ public class IngestManager {
*
* @param moduleDataEvent
*/
static synchronized void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
try {
pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null);
} catch (Exception e) {
@ -199,7 +243,7 @@ public class IngestManager {
*
* @param moduleContentEvent
*/
static synchronized void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
try {
pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null);
} catch (Exception e) {
@ -210,152 +254,6 @@ public class IngestManager {
}
}
/**
* Multiple data-sources version of scheduleDataSource() method. Enqueues
* multiple sources inputs (Content objects) and associated modules at once
*
* @param modules modules to scheduleDataSource on every data source
* @param inputs input data sources to enqueue and scheduleDataSource the
* ingest modules on
*/
void scheduleDataSourceTasks(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
if (!isIngestRunning() && ingestMessageBox != null) {
ingestMessageBox.clearMessages();
}
taskSchedulingWorker = new TaskSchedulingWorker(dataSources, moduleTemplates, processUnallocatedSpace);
taskSchedulingWorker.execute();
if (ingestMessageBox != null) {
ingestMessageBox.restoreMessages();
}
}
/**
* IngestManager entry point, enqueues data to be processed and starts new
* ingest as needed, or just enqueues data to an existing pipeline.
*
* Spawns background thread which enumerates all sorted files and executes
* chosen modules per file in a pre-determined order. Notifies modules when
* work is complete or should be interrupted using complete() and stop()
* calls. Does not block and can be called multiple times to enqueue more
* work to already running background ingest process.
*
* @param modules modules to scheduleDataSource on the data source input
* @param input input data source Content objects to scheduleDataSource the
* ingest modules on
*/
void scheduleDataSourceTask(final Content dataSource, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
List<Content> dataSources = new ArrayList<>();
dataSources.add(dataSource);
scheduleDataSourceTasks(dataSources, moduleTemplates, processUnallocatedSpace);
}
/**
* Schedule a file for ingest and add it to ongoing file ingest process on
* the same data source. Scheduler updates the current progress.
*
* The file to be added is usually a product of a currently ran ingest. Now
* we want to process this new file with the same ingest context.
*
* @param file file to be scheduled
* @param pipelineContext ingest context used to ingest parent of the file
* to be scheduled
*/
void scheduleFile(long ingestJobId, AbstractFile file) {
IngestJob job = this.ingestJobs.get(ingestJobId);
if (job == null) {
logger.log(Level.SEVERE, "Unable to map ingest job id (id = {0}) to an ingest job, failed to schedule file (id = {1})", new Object[]{ingestJobId, file.getId()});
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
"Unable to associate " + file.getName() + " with ingest job, file will not be processed by ingest nodules",
MessageNotifyUtil.MessageType.ERROR);
}
scheduler.getFileScheduler().scheduleFile(job, file);
}
/**
* Starts the File-level Ingest Module pipeline and the Data Source-level
* Ingest Modules for the queued up data sources and files.
*
* if AbstractFile module is still running, do nothing and allow it to
* consume queue otherwise start /restart AbstractFile worker
*
* data source ingest workers run per (module,content). Checks if one for
* the same (module,content) is already running otherwise start/restart the
* worker
*/
private synchronized void startAll() {
if (!ingestMonitor.isRunning()) {
ingestMonitor.start();
}
if (scheduler.getDataSourceScheduler().hasNext()) {
if (dataSourceTaskWorker == null || dataSourceTaskWorker.isDone()) {
dataSourceTaskWorker = new DataSourceTaskWorker(getNextThreadId());
dataSourceTaskWorker.execute();
}
}
if (scheduler.getFileScheduler().hasNext()) {
if (fileTaskWorker == null || fileTaskWorker.isDone()) {
fileTaskWorker = new FileTaskWorker(getNextThreadId());
fileTaskWorker.execute();
}
}
}
synchronized void reportThreadDone(long threadId) {
for (IngestJob job : ingestJobs.values()) {
job.releaseIngestPipelinesForThread(threadId);
if (job.areIngestPipelinesShutDown()) {
ingestJobs.remove(job.getId());
}
}
}
synchronized void stopAll() {
// First get the task scheduling worker to stop.
if (taskSchedulingWorker != null) {
taskSchedulingWorker.cancel(true);
while (!taskSchedulingWorker.isDone()) {
// Wait.
}
taskSchedulingWorker = null;
}
// Now mark all of the ingest jobs as cancelled. This way the ingest
// modules will know they are being shut down due to cancellation when
// the ingest worker threads release their pipelines.
for (IngestJob job : ingestJobs.values()) {
job.cancel();
}
// Cancel the worker threads.
if (dataSourceTaskWorker != null) {
dataSourceTaskWorker.cancel(true);
}
if (fileTaskWorker != null) {
fileTaskWorker.cancel(true);
}
// Jettision the remaining tasks. This will dispose of any tasks that
// the scheduling worker queued up before it was cancelled.
scheduler.getFileScheduler().empty();
scheduler.getDataSourceScheduler().empty();
}
/**
* Test if any ingest modules are running
*
* @return true if any module is running, false otherwise
*/
public synchronized boolean isIngestRunning() {
return ((taskSchedulingWorker != null && !taskSchedulingWorker.isDone())
|| (fileTaskWorker != null && !fileTaskWorker.isDone())
|| (fileTaskWorker != null && !fileTaskWorker.isDone()));
}
/**
* Module publishes message using InegestManager handle Does not block. The
* message gets enqueued in the GUI thread and displayed in a widget
@ -384,82 +282,219 @@ public class IngestManager {
}
}
private class TaskSchedulingWorker extends SwingWorker<Object, Void> {
private synchronized void startIngestTasks() {
if (!ingestMonitor.isRunning()) {
ingestMonitor.start();
}
long taskId = ingestTaskId.incrementAndGet();
Future<?> task = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
int numberOfFileTasksRequested = getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfFileTasksRequested; ++i) {
taskId = ingestTaskId.incrementAndGet();
task = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
}
}
private synchronized void stopIngestTasks() {
// First mark all of the ingest jobs as cancelled. This way the
// ingest modules will know they are being shut down due to
// cancellation when the cancelled run ingest module tasks release
// their pipelines.
for (IngestJob job : ingestJobs.values()) {
job.cancel();
}
// Cancel the run ingest module tasks, setting the state of the threads
// running them to interrupted.
for (Future<?> task : ingestTasks.values()) {
task.cancel(true);
}
// Jettision the remaining data source and file ingest tasks.
scheduler.getFileScheduler().empty();
scheduler.getDataSourceScheduler().empty();
}
synchronized void reportStartIngestJobsTaskDone(long taskId) {
ingestTasks.remove(taskId);
}
synchronized void reportRunIngestModulesTaskDone(long taskId) {
ingestTasks.remove(taskId);
List<Long> completedJobs = new ArrayList<>();
for (IngestJob job : ingestJobs.values()) {
job.releaseIngestPipelinesForThread(taskId);
if (job.areIngestPipelinesShutDown() == true) {
completedJobs.add(job.getId());
}
}
for (Long jobId : completedJobs) {
IngestJob job = ingestJobs.remove(jobId);
fireIngestJobEvent(job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED.toString() : IngestEvent.INGEST_JOB_COMPLETED.toString(), jobId);
}
}
private class StartIngestJobsTask implements Runnable {
private final long id;
private final List<Content> dataSources;
private final List<IngestModuleTemplate> moduleTemplates;
private final boolean processUnallocatedSpace;
private ProgressHandle progress;
TaskSchedulingWorker(List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
StartIngestJobsTask(long taskId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
this.id = taskId;
this.dataSources = dataSources;
this.moduleTemplates = moduleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace;
}
@Override
protected Object doInBackground() throws Exception {
// Set up a progress bar that can be used to cancel all of the
// ingest jobs currently being performed.
final String displayName = "Queueing ingest tasks";
progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
logger.log(Level.INFO, "Queueing ingest cancelled by user.");
if (progress != null) {
progress.setDisplayName(displayName + " (Cancelling...)");
}
IngestManager.getInstance().stopAll();
return true;
}
});
progress.start(2 * dataSources.size());
int processed = 0;
for (Content dataSource : dataSources) {
if (isCancelled()) {
logger.log(Level.INFO, "Task scheduling thread cancelled");
return null;
}
final String inputName = dataSource.getName();
IngestJob ingestJob = new IngestJob(IngestManager.this.getNextDataSourceTaskId(), dataSource, moduleTemplates, processUnallocatedSpace);
List<IngestModuleError> errors = ingestJob.startUpIngestPipelines();
if (!errors.isEmpty()) {
StringBuilder failedModules = new StringBuilder();
for (int i = 0; i < errors.size(); ++i) {
IngestModuleError error = errors.get(i);
String moduleName = error.getModuleDisplayName();
logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError());
failedModules.append(moduleName);
if ((errors.size() > 1) && (i != (errors.size() - 1))) {
failedModules.append(",");
public void run() {
try {
final String displayName = "Queueing ingest tasks";
progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
if (progress != null) {
progress.setDisplayName(displayName + " (Cancelling...)");
}
IngestManager.getInstance().cancelIngestJobs();
return true;
}
});
progress.start(2 * dataSources.size());
int workUnitsCompleted = 0;
for (Content dataSource : dataSources) {
if (Thread.currentThread().isInterrupted()) {
break;
}
IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace);
List<IngestModuleError> errors = ingestJob.startUpIngestPipelines();
if (!errors.isEmpty()) {
StringBuilder failedModules = new StringBuilder();
for (int i = 0; i < errors.size(); ++i) {
IngestModuleError error = errors.get(i);
String moduleName = error.getModuleDisplayName();
logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError());
failedModules.append(moduleName);
if ((errors.size() > 1) && (i != (errors.size() - 1))) {
failedModules.append(",");
}
}
MessageNotifyUtil.Message.error( // RJCTODO: Fix this to show all errors, probably should specify data source name
"Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n"
+ "No ingest modules will be run. Please disable the module "
+ "or fix the error and restart ingest by right clicking on "
+ "the data source and selecting Run Ingest Modules.\n\n"
+ "Error: " + errors.get(0).getModuleError().getMessage());
ingestJob.cancel();
break;
}
// Save the ingest job for later cleanup of pipelines.
synchronized (IngestManager.this) {
ingestJobs.put(ingestJob.getId(), ingestJob);
}
// Queue the data source ingest tasks for the ingest job.
final String inputName = dataSource.getName();
progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); // RJCTODO: Improve
scheduler.getDataSourceScheduler().schedule(ingestJob);
progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted);
// Queue the file ingest tasks for the ingest job.
progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted);
scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob);
progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted);
if (!Thread.currentThread().isInterrupted()) {
startIngestTasks();
fireIngestJobEvent(IngestEvent.INGEST_JOB_STARTED.toString(), ingestJob.getId());
}
MessageNotifyUtil.Message.error(
"Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n"
+ "No ingest modules will be run. Please disable the module "
+ "or fix the error and restart ingest by right clicking on "
+ "the data source and selecting Run Ingest Modules.\n\n"
+ "Error: " + errors.get(0).getModuleError().getMessage());
return null;
}
// Save the ingest job for later cleanup of pipelines.
ingestJobs.put(ingestJob.getId(), ingestJob);
// Queue the data source ingest tasks for the ingest job.
progress.progress("DataSource Ingest" + " " + inputName, processed);
scheduler.getDataSourceScheduler().schedule(ingestJob);
progress.progress("DataSource Ingest" + " " + inputName, ++processed);
// Queue the file ingest tasks for the ingest job.
progress.progress("File Ingest" + " " + inputName, processed);
scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob);
progress.progress("File Ingest" + " " + inputName, ++processed);
} catch (Exception ex) {
String message = String.format("StartIngestJobsTask (id=%d) caught exception", id);
logger.log(Level.SEVERE, message, ex);
MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial");
} finally {
progress.finish();
reportStartIngestJobsTaskDone(id);
}
}
}
private class RunDataSourceIngestModulesTask implements Runnable {
private final long id;
RunDataSourceIngestModulesTask(long taskId) {
id = taskId;
}
@Override
public void run() {
try {
IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler();
while (scheduler.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
break;
}
IngestJob job = scheduler.next();
job.getDataSourceIngestPipelineForThread(id).process();
}
} catch (Exception ex) {
String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id);
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
}
}
}
private class RunFileSourceIngestModulesTask implements Runnable {
private final long id;
RunFileSourceIngestModulesTask(long taskId) {
id = taskId;
}
@Override
public void run() {
try {
IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler();
while (fileScheduler.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
break;
}
IngestScheduler.FileScheduler.FileTask task = fileScheduler.next();
IngestJob job = task.getJob();
job.updateFileTasksProgressBar(task.getFile().getName());
job.getFileIngestPipelineForThread(id).process(task.getFile());
}
} catch (Exception ex) {
String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id);
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
}
}
}
class IngestCancellationWorker extends SwingWorker<Void, Void> {
@Override
protected Void doInBackground() throws Exception {
stopIngestTasks();
return null;
}
@ -468,172 +503,8 @@ public class IngestManager {
try {
super.get();
} catch (CancellationException | InterruptedException ex) {
// IngestManager.stopAll() will dispose of all tasks.
} catch (Exception ex) {
logger.log(Level.SEVERE, "Error while scheduling ingest jobs", ex);
MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial");
} finally {
if (!isCancelled()) {
startAll();
}
progress.finish();
}
}
}
/**
* Performs data source ingest tasks for one or more ingest jobs on a worker
* thread.
*/
class DataSourceTaskWorker extends SwingWorker<Object, Void> {
private final long id;
private ProgressHandle progress;
DataSourceTaskWorker(long threadId) {
this.id = threadId;
}
@Override
protected Void doInBackground() throws Exception {
logger.log(Level.INFO, "Data source ingest thread (id={0}) started", this.id);
// Set up a progress bar that can be used to cancel all of the
// ingest jobs currently being performed.
progress = ProgressHandleFactory.createHandle("Data source ingest", new Cancellable() {
@Override
public boolean cancel() {
logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", DataSourceTaskWorker.this.id);
if (progress != null) {
progress.setDisplayName(NbBundle.getMessage(this.getClass(),
"IngestManager.DataSourceTaskWorker.process.cancelling",
"Data source ingest"));
}
IngestManager.getInstance().stopAll();
return true;
}
});
progress.start();
progress.switchToIndeterminate();
IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler();
while (scheduler.hasNext()) {
if (isCancelled()) {
logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id);
return null;
}
IngestJob ingestJob = scheduler.next();
IngestJob.DataSourceIngestPipeline pipeline = ingestJob.getDataSourceIngestPipelineForThread(this.id);
pipeline.process(this, this.progress);
}
logger.log(Level.INFO, "Data source ingest thread (id={0}) completed", this.id);
IngestManager.getInstance().reportThreadDone(this.id);
return null;
}
@Override
protected void done() {
try {
super.get();
} catch (CancellationException | InterruptedException e) {
logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id);
IngestManager.getInstance().reportThreadDone(this.id);
} catch (Exception ex) {
String message = String.format("Data source ingest thread (id=%d) experienced a fatal error", this.id);
logger.log(Level.SEVERE, message, ex);
IngestManager.getInstance().reportThreadDone(this.id);
} finally {
progress.finish();
}
}
}
/**
* Performs file ingest tasks for one or more ingest jobs on a worker
* thread.
*/
class FileTaskWorker extends SwingWorker<Object, Void> {
private final long id;
private ProgressHandle progress;
FileTaskWorker(long threadId) {
this.id = threadId;
}
@Override
protected Object doInBackground() throws Exception {
logger.log(Level.INFO, "File ingest thread (id={0}) started", this.id);
// Set up a progress bar that can be used to cancel all of the
// ingest jobs currently being performed.
final String displayName = NbBundle
.getMessage(this.getClass(), "IngestManager.FileTaskWorker.displayName");
progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", FileTaskWorker.this.id);
if (progress != null) {
progress.setDisplayName(
NbBundle.getMessage(this.getClass(), "IngestManager.FileTaskWorker.process.cancelling",
displayName));
}
IngestManager.getInstance().stopAll();
return true;
}
});
progress.start();
progress.switchToIndeterminate();
IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler();
int totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
progress.switchToDeterminate(totalEnqueuedFiles);
int processedFiles = 0;
while (fileScheduler.hasNext()) {
if (isCancelled()) {
IngestManager.getInstance().reportThreadDone(this.id);
logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id);
return null;
}
IngestScheduler.FileScheduler.FileTask task = fileScheduler.next();
AbstractFile file = task.getFile();
progress.progress(file.getName(), processedFiles);
IngestJob.FileIngestPipeline pipeline = task.getJob().getFileIngestPipelineForThread(this.id);
pipeline.process(file);
// Update the progress bar.
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
totalEnqueuedFiles = newTotalEnqueuedFiles + 1;
progress.switchToIndeterminate();
progress.switchToDeterminate(totalEnqueuedFiles);
}
if (processedFiles < totalEnqueuedFiles) {
++processedFiles;
}
}
logger.log(Level.INFO, "File ingest thread (id={0}) completed", this.id);
IngestManager.getInstance().reportThreadDone(this.id);
return null;
}
@Override
protected void done() {
try {
super.get();
} catch (CancellationException | InterruptedException e) {
logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id);
IngestManager.getInstance().reportThreadDone(this.id);
} catch (Exception ex) {
String message = String.format("File ingest thread {0} experienced a fatal error", this.id);
logger.log(Level.SEVERE, message, ex);
IngestManager.getInstance().reportThreadDone(this.id);
} finally {
progress.finish();
logger.log(Level.SEVERE, "Error while cancelling ingest jobs", ex);
}
}
}

22
Core/src/org/sleuthkit/autopsy/ingest/IngestMessage.java
View File

@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.ingest;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.atomic.AtomicLong;
import org.openide.util.NbBundle;
import org.sleuthkit.datamodel.BlackboardArtifact;
@ -48,7 +49,8 @@ public class IngestMessage {
private Date datePosted;
private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static int managerMessageId = 0;
private static AtomicLong nextMessageID = new AtomicLong(0);
/**
* Private constructor used by factory methods
*/
@ -175,11 +177,12 @@ public class IngestMessage {
* @param detailsHtml html formatted detailed message (without leading and closing &lt;html&gt; tags), for instance, a human-readable representation of the data. Or null.
* @return
*/
public static IngestMessage createMessage(long ID, MessageType messageType, String source, String subject, String detailsHtml) {
public static IngestMessage createMessage(MessageType messageType, String source, String subject, String detailsHtml) {
if (messageType == null || source == null || subject == null) {
throw new IllegalArgumentException(
NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.typeSrcSubjNotNull.msg"));
}
long ID = nextMessageID.getAndIncrement();
return new IngestMessage(ID, messageType, source, subject, detailsHtml, null);
}
@ -191,8 +194,8 @@ public class IngestMessage {
* @param subject message subject to be displayed
* @return
*/
public static IngestMessage createMessage(long ID, MessageType messageType, String source, String subject) {
return createMessage(ID, messageType, source, subject, null);
public static IngestMessage createMessage(MessageType messageType, String source, String subject) {
return createMessage(messageType, source, subject, null);
}
@ -204,11 +207,12 @@ public class IngestMessage {
* @param detailsHtml html formatted detailed message (without leading and closing &lt;html&gt; tags), for instance, a human-readable representation of the data. Or null
* @return
*/
public static IngestMessage createErrorMessage(long ID, String source, String subject, String detailsHtml) {
public static IngestMessage createErrorMessage(String source, String subject, String detailsHtml) {
if (source == null || subject == null) {
throw new IllegalArgumentException(
NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.srcSubjNotNull.msg"));
}
long ID = nextMessageID.getAndIncrement();
return new IngestMessage(ID, MessageType.ERROR, source, subject, detailsHtml, null);
}
@ -220,11 +224,12 @@ public class IngestMessage {
* @param detailsHtml html formatted detailed message (without leading and closing &lt;html&gt; tags), for instance, a human-readable representation of the data. Or null
* @return
*/
public static IngestMessage createWarningMessage(long ID, String source, String subject, String detailsHtml) {
public static IngestMessage createWarningMessage(String source, String subject, String detailsHtml) {
if (source == null || subject == null) {
throw new IllegalArgumentException(
NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.srcSubjNotNull.msg"));
}
long ID = nextMessageID.getAndIncrement();
return new IngestMessage(ID, MessageType.WARNING, source, subject, detailsHtml, null);
}
@ -238,12 +243,13 @@ public class IngestMessage {
* @param data blackboard artifact associated with the message, the same as fired in ModuleDataEvent by the module
* @return
*/
public static IngestMessage createDataMessage(long ID, String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data) {
public static IngestMessage createDataMessage(String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data) {
if (source == null || subject == null || detailsHtml == null || data == null) {
throw new IllegalArgumentException(
NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.srcSubjDetailsDataNotNull.msg"));
}
long ID = nextMessageID.getAndIncrement();
IngestMessage im = new IngestMessage(ID, MessageType.DATA, source, subject, detailsHtml, uniqueKey);
im.data = data;
return im;

126
Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java
View File

@ -395,18 +395,18 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
}
@Override
synchronized public int getRowCount() {
public synchronized int getRowCount() {
return getNumberGroups();
}
public void markAllSeen() {
public synchronized void markAllSeen() {
for (TableEntry entry : messageData) {
entry.hasBeenSeen(true);
}
fireTableChanged(new TableModelEvent(this));
}
public int getNumberNewMessages() {
public synchronized int getNumberNewMessages() {
int newMessages = 0;
for (TableEntry entry : messageData) {
if (!entry.hasBeenSeen()) {
@ -416,11 +416,11 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return newMessages;
}
synchronized int getNumberGroups() {
public synchronized int getNumberGroups() {
return messageData.size();
}
synchronized int getNumberMessages() {
public synchronized int getNumberMessages() {
int total = 0;
for (TableEntry e : messageData) {
total += e.messageGroup.getCount();
@ -428,7 +428,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return total;
}
synchronized int getNumberUnreadMessages() {
public synchronized int getNumberUnreadMessages() {
int total = 0;
for (TableEntry e : messageData) {
if (!e.hasBeenVisited) {
@ -438,7 +438,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return total;
}
synchronized int getNumberUnreadGroups() {
public synchronized int getNumberUnreadGroups() {
int total = 0;
for (TableEntry e : messageData) {
if (!e.hasBeenVisited) {
@ -513,7 +513,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return ret;
}
synchronized public void addMessage(IngestMessage m) {
public synchronized void addMessage(IngestMessage m) {
//check how many messages per module with the same uniqness
//and add to existing group or create a new group
String moduleName = m.getSource();
@ -628,15 +628,27 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
}
public synchronized boolean isVisited(int rowNumber) {
return messageData.get(rowNumber).hasBeenVisited();
if (rowNumber < messageData.size()) {
return messageData.get(rowNumber).hasBeenVisited();
} else {
return false;
}
}
public synchronized MessageType getMessageType(int rowNumber) {
return messageData.get(rowNumber).messageGroup.getMessageType();
if (rowNumber < messageData.size()) {
return messageData.get(rowNumber).messageGroup.getMessageType();
} else {
return null;
}
}
public synchronized IngestMessageGroup getMessageGroup(int rowNumber) {
return messageData.get(rowNumber).messageGroup;
if (rowNumber < messageData.size()) {
return messageData.get(rowNumber).messageGroup;
} else {
return null;
}
}
public synchronized void reSort(boolean chronoLogical) {
@ -701,26 +713,26 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
messages.add(message);
}
List<IngestMessage> getMessages() {
private List<IngestMessage> getMessages() {
return messages;
}
void add(IngestMessage message) {
}
synchronized void add(IngestMessage message) {
messages.add(message);
}
//add all messages from another group
void addAll(IngestMessageGroup group) {
synchronized void addAll(IngestMessageGroup group) {
for (IngestMessage m : group.getMessages()) {
messages.add(m);
}
}
int getCount() {
synchronized int getCount() {
return messages.size();
}
String getDetails() {
synchronized String getDetails() {
StringBuilder b = new StringBuilder("");
for (IngestMessage m : messages) {
String details = m.getDetails();
@ -739,7 +751,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
* return color corresp to priority
* @return
*/
Color getColor() {
synchronized Color getColor() {
int count = messages.size();
if (count == 1) {
return VERY_HIGH_PRI_COLOR;
@ -757,7 +769,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
* used for chrono sort
* @return
*/
Date getDatePosted() {
synchronized Date getDatePosted() {
return messages.get(messages.size() - 1).getDatePosted();
}
@ -765,35 +777,35 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
* get subject of the first message
* @return
*/
String getSubject() {
synchronized String getSubject() {
return messages.get(0).getSubject();
}
/*
* return unique key, should be the same for all msgs
*/
String getUniqueKey() {
synchronized String getUniqueKey() {
return messages.get(0).getUniqueKey();
}
/*
* return source module, should be the same for all msgs
*/
String getSource() {
synchronized String getSource() {
return messages.get(0).getSource();
}
/*
* return data of the first message
*/
BlackboardArtifact getData() {
synchronized BlackboardArtifact getData() {
return messages.get(0).getData();
}
/*
* return message type, should be the same for all msgs
*/
IngestMessage.MessageType getMessageType() {
synchronized IngestMessage.MessageType getMessageType() {
return messages.get(0).getMessageType();
}
}
@ -858,16 +870,17 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
cell.setFont(new Font("", Font.PLAIN, 16));
final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row);
MessageType mt = messageGroup.getMessageType();
if (mt == MessageType.ERROR) {
cell.setBackground(ERROR_COLOR);
} else if (mt == MessageType.WARNING) {
cell.setBackground(Color.orange);
} else {
//cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
if (messageGroup != null) {
MessageType mt = messageGroup.getMessageType();
if (mt == MessageType.ERROR) {
cell.setBackground(ERROR_COLOR);
} else if (mt == MessageType.WARNING) {
cell.setBackground(Color.orange);
} else {
//cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
}
}
return cell;
}
}
@ -898,16 +911,17 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
}
final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row);
MessageType mt = messageGroup.getMessageType();
if (mt == MessageType.ERROR) {
cell.setBackground(ERROR_COLOR);
} else if (mt == MessageType.WARNING) {
cell.setBackground(Color.orange);
} else {
//cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
if (messageGroup != null) {
MessageType mt = messageGroup.getMessageType();
if (mt == MessageType.ERROR) {
cell.setBackground(ERROR_COLOR);
} else if (mt == MessageType.WARNING) {
cell.setBackground(Color.orange);
} else {
//cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
}
}
return cell;
}
}
@ -933,14 +947,16 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
Component cell = super.getTableCellRendererComponent(table, aValue, isSelected, hasFocus, row, column);
final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row);
MessageType mt = messageGroup.getMessageType();
if (mt == MessageType.ERROR) {
cell.setBackground(ERROR_COLOR);
} else if (mt == MessageType.WARNING) {
cell.setBackground(Color.orange);
} else {
//cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
if (messageGroup != null) {
MessageType mt = messageGroup.getMessageType();
if (mt == MessageType.ERROR) {
cell.setBackground(ERROR_COLOR);
} else if (mt == MessageType.WARNING) {
cell.setBackground(Color.orange);
} else {
//cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
}
}
return cell;
@ -974,9 +990,11 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
messageTable.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
//check if has details
IngestMessageGroup m = getMessageGroup(selected);
String details = m.getDetails();
if (details != null && !details.equals("")) {
mainPanel.showDetails(selected);
if (m != null) {
String details = m.getDetails();
if (details != null && !details.equals("")) {
mainPanel.showDetails(selected);
}
}
messageTable.setCursor(null);
}

2
Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java
View File

@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content;
manager = IngestManager.getInstance();
}
try {
manager.stopAll();
manager.cancelIngestJobs();
} finally {
//clear inbox
clearMessages();

80
Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java
View File

@ -20,27 +20,29 @@ package org.sleuthkit.autopsy.ingest;
/**
* The interface that must be implemented by all ingest modules.
* <p>
*
* Autopsy will generally use several instances of an ingest module for each
* ingest job it performs. Completing an ingest job entails processing a single
* data source (e.g., a disk image) and all of the files from the data source,
* including files extracted from archives and any unallocated space (made to
* look like a series of files). The data source is passed through one or more
* pipelines of data source ingest modules. The files are passed through one or
* more pipelines of file ingest modules.
* <p>
* ingest job it performs (one for each thread that it is using).
*
* Autopsy will call startUp() before any data is processed, will pass any
* data to be analyzed into the process() method (FileIngestModule.process() or DataSourceIngestModule.process()),
* and call shutDown() after
* either all data is analyzed or the has has cancelled the job.
*
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that there will be no more than one module instance per thread.
* However, if the module instances must share resources, the modules are
* guaranteed that a module instance will always be called from a single thread.
* Therefore, you can easily have thread-safe code by not using any static
* member variables.
*
* If the module instances must share resources, the modules are
* responsible for synchronizing access to the shared resources and doing
* reference counting as required to release those resources correctly. Also,
* more than one ingest job may be in progress at any given time. This must also
* be taken into consideration when sharing resources between module instances.
* <p>
*
* TIP: An ingest module that does not require initialization or clean up may
* extend the abstract IngestModuleAdapter class to get a default "do nothing"
* implementation of this interface.
*
*/
public interface IngestModule {
@ -70,27 +72,13 @@ public interface IngestModule {
* Invoked by Autopsy to allow an ingest module instance to set up any
* internal data structures and acquire any private resources it will need
* during an ingest job.
* <p>
* Autopsy will generally use several instances of an ingest module for each
* ingest job it performs. Completing an ingest job entails processing a
* single data source (e.g., a disk image) and all of the files from the
* data source, including files extracted from archives and any unallocated
* space (made to look like a series of files). The data source is passed
* through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
* <p>
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that there will be no more than one module instance per
* thread. However, if the module instances must share resources, the
* modules are responsible for synchronizing access to the shared resources
* and doing reference counting as required to release those resources
* correctly. Also, more than one ingest job may be in progress at any given
* time. This must also be taken into consideration when sharing resources
* between module instances.
* <p>
* An ingest module that does not require initialization may extend the
* abstract IngestModuleAdapter class to get a default "do nothing"
* implementation of this method.
*
* If the module depends on loading any resources, it should do so in this
* method so that it can throw an exception in the case of an error and
* alert the user. Exceptions that are thrown from process() and shutDown()
* are logged, but do not stop processing of the data source.
*
* On error, throw a IngestModuleException.
*
* @param context Provides data and services specific to the ingest job and
* the ingest pipeline of which the module is a part.
@ -99,31 +87,13 @@ public interface IngestModule {
void startUp(IngestJobContext context) throws IngestModuleException;
/**
* Invoked by Autopsy when an ingest job is completed, before the ingest
* Invoked by Autopsy when an ingest job is completed (either because the
* data has been analyzed or because the job was cancelled), before the ingest
* module instance is discarded. The module should respond by doing things
* like releasing private resources, submitting final results, and posting a
* final ingest message.
* <p>
* Autopsy will generally use several instances of an ingest module for each
* ingest job it performs. Completing an ingest job entails processing a
* single data source (e.g., a disk image) and all of the files from the
* data source, including files extracted from archives and any unallocated
* space (made to look like a series of files). The data source is passed
* through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
* <p>
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that there will be no more than one module instance per
* thread. However, if the module instances must share resources, the
* modules are responsible for synchronizing access to the shared resources
* and doing reference counting as required to release those resources
* correctly. Also, more than one ingest job may be in progress at any given
* time. This must also be taken into consideration when sharing resources
* between module instances.
* <p>
* An ingest module that does not require initialization may extend the
* abstract IngestModuleAdapter class to get a default "do nothing"
* implementation of this method.
* @param ingestJobWasCancelled True if this is being called because the user
* cancelled the job.
*/
void shutDown(boolean ingestJobWasCancelled);
}

1
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java
View File

@ -23,7 +23,6 @@ package org.sleuthkit.autopsy.ingest;
* interface.
*/
public abstract class IngestModuleAdapter implements IngestModule {
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
}

10
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java
View File

@ -23,10 +23,10 @@ package org.sleuthkit.autopsy.ingest;
* modules. An ingest module factory is used to create instances of a type of
* data source ingest module, a type of file ingest module, or both.
* <p>
* Autopsy will generally use the factory to several instances of each type of
* module for each ingest job it performs. Completing an ingest job entails
* processing a single data source (e.g., a disk image) and all of the files
* from the data source, including files extracted from archives and any
* Autopsy will generally use the factory to create several instances of each
* type of module for each ingest job it performs. Completing an ingest job
* entails processing a single data source (e.g., a disk image) and all of the
* files from the data source, including files extracted from archives and any
* unallocated space (made to look like a series of files). The data source is
* passed through one or more pipelines of data source ingest modules. The files
* are passed through one or more pipelines of file ingest modules.
@ -52,7 +52,7 @@ package org.sleuthkit.autopsy.ingest;
* @ServiceProvider(service=IngestModuleFactory.class)
* <p>
* IMPORTANT TIP: If an implementation of IngestModuleFactory does not need to
* provide implementations of all of the IngestModuleFactory methods, it can
* provide implementations of all of the IngestModuleFactory methods, it can
* extend the abstract class IngestModuleFactoryAdapter to get default
* implementations of most of the IngestModuleFactory methods.
*/

48
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java
View File

@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.logging.Level;
@ -27,7 +28,7 @@ import org.openide.util.Lookup;
import org.sleuthkit.autopsy.coreutils.Logger;
/**
* Looks up loaded ingest module factories using the NetBean global lookup.
* Looks up loaded ingest module factories using the NetBeans global lookup.
*/
final class IngestModuleFactoryLoader {
@ -43,20 +44,57 @@ final class IngestModuleFactoryLoader {
}
return instance;
}
synchronized List<IngestModuleFactory> getIngestModuleFactories() {
List<IngestModuleFactory> moduleFactories = new ArrayList<>();
// Discover the ingest module factories, making sure that there are no
// duplicate module display names. The duplicates requirement could be
// eliminated if the enabled/disabled modules setting was by factory
// class name instead of module display name. Also note that that we are
// temporarily hard-coding ordering of module factories until the
// module configuration file is reworked, so the discovered factories
// are initially mapped by class name.
HashSet<String> moduleDisplayNames = new HashSet<>();
HashMap<String, IngestModuleFactory> moduleFactoriesByClass = new HashMap<>();
Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class);
for (IngestModuleFactory factory : factories) {
logger.log(Level.INFO, "Found ingest module factory: name = {0}, version = {1}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()});
if (!moduleDisplayNames.contains(factory.getModuleDisplayName())) {
moduleFactories.add(factory);
moduleFactoriesByClass.put(factory.getClass().getCanonicalName(), factory);
moduleDisplayNames.add(factory.getModuleDisplayName());
} else {
// Not popping up a message box to keep this class UI-indepdent.
logger.log(Level.SEVERE, "Found duplicate ingest module display name, discarding ingest module factory (name = {0}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()});
}
}
return new ArrayList<>(moduleFactories);
// Kick out the sample modules factory.
moduleFactoriesByClass.remove("org.sleuthkit.autopsy.examples.SampleIngestModuleFactory");
// Do the core ingest module ordering hack described above.
ArrayList<String> coreModuleOrdering = new ArrayList<String>() {{
add("org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory");
add("org.sleuthkit.autopsy.ewfverify.EwfVerifierModuleFactory");
add("org.sleuthkit.autopsy.hashdatabase.HashLookupModuleFactory");
add("org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory");
add("org.sleuthkit.autopsy.modules.sevenzip.ArchiveFileExtractorModuleFactory");
add("org.sleuthkit.autopsy.modules.exif.ExifParserModuleFactory");
add("org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleFactory");
add("org.sleuthkit.autopsy.thunderbirdparser.EmailParserModuleFactory");
add("org.sleuthkit.autopsy.fileextmismatch.FileExtMismatchDetectorModuleFactory");
}};
List<IngestModuleFactory> orderedModuleFactories = new ArrayList<>();
for (String className : coreModuleOrdering) {
IngestModuleFactory coreFactory = moduleFactoriesByClass.remove(className);
if (coreFactory != null) {
orderedModuleFactories.add(coreFactory);
}
}
// Add in any non-core factories discovered. Order is not guaranteed!
for (IngestModuleFactory nonCoreFactory : moduleFactoriesByClass.values()) {
orderedModuleFactories.add(nonCoreFactory);
}
return orderedModuleFactories;
}
}

59
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleReferenceCounter.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011 - 2013 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.HashMap;
/**
* A utility class that modules can use to keep track of whether they are the
* first/last instance for a particular job.
*
* An instance of this should be static in your module class.
*/
public class IngestModuleReferenceCounter {
// Maps a JobId to the count of instances
private HashMap<Long, Long> moduleRefCount = new HashMap<>();
public synchronized long get(long jobId) {
return moduleRefCount.get(jobId);
}
public synchronized long incrementAndGet(long jobId) {
long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0;
long nextCount = count + 1;
moduleRefCount.put(jobId, nextCount);
return nextCount;
}
public synchronized long decrementAndGet(long jobId) {
if (moduleRefCount.containsKey(jobId)) {
long count = moduleRefCount.get(jobId);
if (--count == 0) {
moduleRefCount.remove(jobId);
} else {
moduleRefCount.put(jobId, count);
}
return count;
} else {
return -1;
}
}
}

6
Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java
View File

@ -163,9 +163,9 @@ public final class IngestMonitor {
if (checkDiskSpace() == false) {
//stop ingest if running
final String diskPath = root.getAbsolutePath();
MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath);
logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath);
manager.stopAll();
MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath);
logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath);
manager.cancelIngestJobs();
IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage(
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath),
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath)));

124
Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java
View File

@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.XMLUtil;
import org.w3c.dom.Document;
@ -39,12 +40,15 @@ import org.w3c.dom.NodeList;
final class IngestPipelinesConfiguration {
private static final Logger logger = Logger.getLogger(IngestPipelinesConfiguration.class.getName());
private final static String PIPELINES_CONFIG_FILE = "pipeline_config.xml";
private final static String PIPELINES_CONFIG_FILE_XSD = "PipelineConfigSchema.xsd";
private static final String PIPELINE_CONFIG_FILE_VERSION_KEY = "PipelineConfigFileVersion";
private static final String PIPELINE_CONFIG_FILE_VERSION_NO_STRING = "1";
private static final int PIPELINE_CONFIG_FILE_VERSION_NO = 1;
private static final String PIPELINES_CONFIG_FILE = "pipeline_config.xml";
private static final String PIPELINES_CONFIG_FILE_XSD = "PipelineConfigSchema.xsd";
private static final String XML_PIPELINE_ELEM = "PIPELINE";
private static final String XML_PIPELINE_TYPE_ATTR = "type";
private final static String DATA_SOURCE_INGEST_PIPELINE_TYPE = "ImageAnalysis";
private final static String FILE_INGEST_PIPELINE_TYPE = "FileAnalysis";
private static final String DATA_SOURCE_INGEST_PIPELINE_TYPE = "ImageAnalysis";
private static final String FILE_INGEST_PIPELINE_TYPE = "FileAnalysis";
private static final String XML_MODULE_ELEM = "MODULE";
private static final String XML_MODULE_CLASS_NAME_ATTR = "location";
private static IngestPipelinesConfiguration instance;
@ -73,65 +77,73 @@ final class IngestPipelinesConfiguration {
private void readPipelinesConfigurationFile() {
try {
PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE);
} catch (IOException ex) {
logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir", ex);
return;
}
String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE;
Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath, PIPELINES_CONFIG_FILE_XSD);
if (doc == null) {
return;
}
boolean overWrite;
if (!ModuleSettings.settingExists(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY)) {
ModuleSettings.setConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY, PIPELINE_CONFIG_FILE_VERSION_NO_STRING);
overWrite = true;
} else {
int versionNumber = Integer.parseInt(ModuleSettings.getConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY));
overWrite = versionNumber < PIPELINE_CONFIG_FILE_VERSION_NO;
// TODO: Migrate user edits
}
PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE, overWrite);
Element rootElement = doc.getDocumentElement();
if (rootElement == null) {
logger.log(Level.SEVERE, "Invalid pipelines config file");
return;
}
NodeList pipelineElements = rootElement.getElementsByTagName(XML_PIPELINE_ELEM);
int numPipelines = pipelineElements.getLength();
if (numPipelines < 1 || numPipelines > 2) {
logger.log(Level.SEVERE, "Invalid pipelines config file");
return;
}
List<String> pipelineConfig = null;
for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) {
Element pipelineElement = (Element) pipelineElements.item(pipelineNum);
String pipelineTypeAttr = pipelineElement.getAttribute(XML_PIPELINE_TYPE_ATTR);
if (pipelineTypeAttr != null) {
switch (pipelineTypeAttr) {
case DATA_SOURCE_INGEST_PIPELINE_TYPE:
pipelineConfig = dataSourceIngestPipelineConfig;
break;
case FILE_INGEST_PIPELINE_TYPE:
pipelineConfig = fileIngestPipelineConfig;
break;
default:
logger.log(Level.SEVERE, "Invalid pipelines config file");
return;
}
String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE;
Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath, PIPELINES_CONFIG_FILE_XSD);
if (doc == null) {
return;
}
// Create an ordered list of class names. The sequence of class
// names defines the sequence of modules in the pipeline.
if (pipelineConfig != null) {
NodeList modulesElems = pipelineElement.getElementsByTagName(XML_MODULE_ELEM);
int numModules = modulesElems.getLength();
if (numModules == 0) {
break;
Element rootElement = doc.getDocumentElement();
if (rootElement == null) {
logger.log(Level.SEVERE, "Invalid pipelines config file");
return;
}
NodeList pipelineElements = rootElement.getElementsByTagName(XML_PIPELINE_ELEM);
int numPipelines = pipelineElements.getLength();
if (numPipelines < 1 || numPipelines > 2) {
logger.log(Level.SEVERE, "Invalid pipelines config file");
return;
}
List<String> pipelineConfig = null;
for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) {
Element pipelineElement = (Element) pipelineElements.item(pipelineNum);
String pipelineTypeAttr = pipelineElement.getAttribute(XML_PIPELINE_TYPE_ATTR);
if (pipelineTypeAttr != null) {
switch (pipelineTypeAttr) {
case DATA_SOURCE_INGEST_PIPELINE_TYPE:
pipelineConfig = dataSourceIngestPipelineConfig;
break;
case FILE_INGEST_PIPELINE_TYPE:
pipelineConfig = fileIngestPipelineConfig;
break;
default:
logger.log(Level.SEVERE, "Invalid pipelines config file");
return;
}
}
for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) {
Element moduleElement = (Element) modulesElems.item(moduleNum);
final String moduleClassName = moduleElement.getAttribute(XML_MODULE_CLASS_NAME_ATTR);
if (moduleClassName != null) {
pipelineConfig.add(moduleClassName);
// Create an ordered list of class names. The sequence of class
// names defines the sequence of modules in the pipeline.
if (pipelineConfig != null) {
NodeList modulesElems = pipelineElement.getElementsByTagName(XML_MODULE_ELEM);
int numModules = modulesElems.getLength();
if (numModules == 0) {
break;
}
for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) {
Element moduleElement = (Element) modulesElems.item(moduleNum);
final String moduleClassName = moduleElement.getAttribute(XML_MODULE_CLASS_NAME_ATTR);
if (moduleClassName != null) {
pipelineConfig.add(moduleClassName);
}
}
}
}
} catch (IOException ex) {
logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir", ex);
}
}
}

8
Core/src/org/sleuthkit/autopsy/ingest/pipeline_config.xml
View File

@ -4,12 +4,12 @@ Contains only the core ingest modules that ship with Autopsy -->
<PIPELINE_CONFIG>
<PIPELINE type="FileAnalysis">
<MODULE order="1" type="plugin" location="org.sleuthkit.autopsy.hashdatabase.HashDbIngestModule" arguments="" />
<MODULE order="2" type="plugin" location="org.sleuthkit.autopsy.filetypeid.FileTypeIdIngestModule" arguments=""/>
<MODULE order="3" type="plugin" location="org.sleuthkit.autopsy.sevenzip.SevenZipIngestModule" arguments="" />
<MODULE order="4" type="plugin" location="org.sleuthkit.autopsy.exifparser.ExifParserFileIngestModule"/>
<MODULE order="2" type="plugin" location="org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdIngestModule" arguments=""/>
<MODULE order="3" type="plugin" location="org.sleuthkit.autopsy.modules.sevenzip.SevenZipIngestModule" arguments="" />
<MODULE order="4" type="plugin" location="org.sleuthkit.autopsy.modules.exifparser.ExifParserFileIngestModule"/>
<MODULE order="5" type="plugin" location="org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule"/>
<MODULE order="6" type="plugin" location="org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule" arguments=""/>
<MODULE order="7" type="plugin" location="org.sleuthkit.autopsy.fileextmismatch.FileExtMismatchIngestModule" arguments=""/>
<MODULE order="7" type="plugin" location="org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchIngestModule" arguments=""/>
</PIPELINE>
<PIPELINE type="ImageAnalysis">

0
ExifParser/src/org/sleuthkit/autopsy/exifparser/Bundle.properties → Core/src/org/sleuthkit/autopsy/modules/exif/Bundle.properties
View File

0
ExifParser/src/org/sleuthkit/autopsy/exifparser/Bundle_ja.properties → Core/src/org/sleuthkit/autopsy/modules/exif/Bundle_ja.properties
View File

33
ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java → Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.exifparser;
package org.sleuthkit.autopsy.modules.exif;
import com.drew.imaging.ImageMetadataReader;
import com.drew.imaging.ImageProcessingException;
@ -32,13 +32,16 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.ImageUtils;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
@ -57,12 +60,21 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem
private static final Logger logger = Logger.getLogger(ExifParserFileIngestModule.class.getName());
private final IngestServices services = IngestServices.getInstance();
private int filesProcessed = 0;
private boolean filesToFire = false;
private AtomicInteger filesProcessed = new AtomicInteger(0);
private volatile boolean filesToFire = false;
private long jobId;
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
ExifParserFileIngestModule() {
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
jobId = context.getJobId();
refCounter.incrementAndGet(jobId);
}
@Override
public ProcessResult process(AbstractFile content) {
//skip unalloc
@ -76,8 +88,8 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem
}
// update the tree every 1000 files if we have EXIF data that is not being being displayed
filesProcessed++;
if ((filesToFire) && (filesProcessed % 1000 == 0)) {
final int filesProcessedValue = filesProcessed.incrementAndGet();
if ((filesToFire) && (filesProcessedValue % 1000 == 0)) {
services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF));
filesToFire = false;
}
@ -187,9 +199,12 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem
@Override
public void shutDown(boolean ingestJobCancelled) {
if (filesToFire) {
//send the final new data event
services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF));
// We only need to check for this final event on the last module per job
if (refCounter.decrementAndGet(jobId) == 0) {
if (filesToFire) {
//send the final new data event
services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF));
}
}
}
}

2
ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserModuleFactory.java → Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserModuleFactory.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.exifparser;
package org.sleuthkit.autopsy.modules.exif;
import org.openide.util.lookup.ServiceProvider;
import org.sleuthkit.autopsy.coreutils.Version;

0
ExifParser/src/org/sleuthkit/autopsy/exifparser/layer.xml → Core/src/org/sleuthkit/autopsy/modules/exif/layer.xml
View File

2
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/AddFileExtensionAction.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import org.openide.util.NbBundle;
import java.awt.event.ActionEvent;

1
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties
View File

@ -34,7 +34,6 @@ FileExtMismatchIngestModule.complete.totalFiles=Total Files Processed
FileExtMismatchIngestModule.complete.svcMsg.text=File Extension Mismatch Results
FileExtMismatchOptionsPanelController.moduleErr=Module Error
FileExtMismatchOptionsPanelController.moduleErr.msg=A module caused an error listening to FileExtMismatchOptionsPanelController updates. See log to determine which module. Some data could be incomplete.
FileExtMismatchModuleSettingsPanel.skipKnownFilesCheckbox.text=Skip known files (NSRL)
FileExtMismatchModuleSettingsPanel.skipTextPlain.text=Skip text files
FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text=Skip files without extensions
FileExtMismatchSettingsPanel.addTypeButton.text=Add Type

0
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle_ja.properties → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle_ja.properties
View File

2
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java
View File

@ -18,7 +18,7 @@
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import java.util.ArrayList;
import java.util.Arrays;

2
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleFactory.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleFactory.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import org.openide.util.NbBundle;
import org.openide.util.lookup.ServiceProvider;

12
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleSettings.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchDetectorModuleSettings.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
@ -25,7 +25,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
*/
final class FileExtMismatchDetectorModuleSettings implements IngestModuleIngestJobSettings {
private boolean skipKnownFiles = false;
private boolean skipFilesWithNoExtension = true;
private boolean skipFilesWithTextPlainMimeType = false;
@ -33,19 +32,10 @@ final class FileExtMismatchDetectorModuleSettings implements IngestModuleIngestJ
}
FileExtMismatchDetectorModuleSettings(boolean skipKnownFiles, boolean skipFilesWithNoExtension, boolean skipFilesWithTextPlainMimeType) {
this.skipKnownFiles = skipKnownFiles;
this.skipFilesWithNoExtension = skipFilesWithNoExtension;
this.skipFilesWithTextPlainMimeType = skipFilesWithTextPlainMimeType;
}
void setSkipKnownFiles(boolean enabled) {
skipKnownFiles = enabled;
}
boolean skipKnownFiles() {
return skipKnownFiles;
}
void setSkipFilesWithNoExtension(boolean enabled) {
skipFilesWithNoExtension = enabled;
}

53
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java
View File

@ -16,13 +16,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
@ -32,13 +33,13 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.FileKnown;
import org.sleuthkit.datamodel.TskException;
/**
@ -47,12 +48,13 @@ import org.sleuthkit.datamodel.TskException;
public class FileExtMismatchIngestModule extends IngestModuleAdapter implements FileIngestModule {
private static final Logger logger = Logger.getLogger(FileExtMismatchIngestModule.class.getName());
private static int messageId = 0; // RJCTODO: This is not thread safe
private final IngestServices services = IngestServices.getInstance();
private final FileExtMismatchDetectorModuleSettings settings;
private HashMap<String, String[]> SigTypeToExtMap = new HashMap<>();
private long processTime = 0;
private long numFiles = 0;
private long jobId;
private static AtomicLong processTime = new AtomicLong(0);
private static AtomicLong numFiles = new AtomicLong(0);
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) {
this.settings = settings;
@ -60,6 +62,8 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
jobId = context.getJobId();
refCounter.incrementAndGet(jobId);
FileExtMismatchXML xmlLoader = FileExtMismatchXML.getDefault();
SigTypeToExtMap = xmlLoader.load();
}
@ -78,17 +82,13 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements
return ProcessResult.OK;
}
if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) {
return ProcessResult.OK;
}
try {
long startTime = System.currentTimeMillis();
boolean mismatchDetected = compareSigTypeToExt(abstractFile);
processTime += (System.currentTimeMillis() - startTime);
numFiles++;
processTime.getAndAdd(System.currentTimeMillis() - startTime);
numFiles.getAndIncrement();
if (mismatchDetected) {
// add artifact
@ -154,19 +154,22 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements
@Override
public void shutDown(boolean ingestJobCancelled) {
StringBuilder detailsSb = new StringBuilder();
detailsSb.append("<table border='0' cellpadding='4' width='280'>");
detailsSb.append("<tr><td>").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("</td></tr>");
detailsSb.append("<tr><td>").append(
NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime"))
.append("</td><td>").append(processTime).append("</td></tr>\n");
detailsSb.append("<tr><td>").append(
NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles"))
.append("</td><td>").append(numFiles).append("</td></tr>\n");
detailsSb.append("</table>");
services.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"FileExtMismatchIngestModule.complete.svcMsg.text"),
detailsSb.toString()));
// We only need to post the summary msg from the last module per job
if (refCounter.decrementAndGet(jobId) == 0) {
StringBuilder detailsSb = new StringBuilder();
detailsSb.append("<table border='0' cellpadding='4' width='280'>");
detailsSb.append("<tr><td>").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("</td></tr>");
detailsSb.append("<tr><td>").append(
NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime"))
.append("</td><td>").append(processTime.get()).append("</td></tr>\n");
detailsSb.append("<tr><td>").append(
NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles"))
.append("</td><td>").append(numFiles.get()).append("</td></tr>\n");
detailsSb.append("</table>");
services.postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"FileExtMismatchIngestModule.complete.svcMsg.text"),
detailsSb.toString()));
}
}
}

19
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.form → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.form
View File

@ -21,7 +21,6 @@
<Group type="103" groupAlignment="0" attributes="0">
<Component id="skipTextPlain" min="-2" max="-2" attributes="0"/>
<Component id="skipNoExtCheckBox" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="skipKnownFilesCheckbox" alignment="0" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace min="0" pref="138" max="32767" attributes="0"/>
</Group>
@ -33,9 +32,7 @@
<Component id="skipNoExtCheckBox" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="skipTextPlain" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="skipKnownFilesCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="28" max="32767" attributes="0"/>
<EmptySpace pref="51" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -45,7 +42,7 @@
<Properties>
<Property name="selected" type="boolean" value="true"/>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -55,7 +52,7 @@
<Component class="javax.swing.JCheckBox" name="skipTextPlain">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchModuleSettingsPanel.skipTextPlain.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchModuleSettingsPanel.skipTextPlain.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -65,15 +62,5 @@
<AuxValue name="JavaCodeGenerator_InitCodePost" type="java.lang.String" value="skipTextPlain.setSelected(true);"/>
</AuxValues>
</Component>
<Component class="javax.swing.JCheckBox" name="skipKnownFilesCheckbox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchModuleSettingsPanel.skipKnownFilesCheckbox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="skipKnownFilesCheckboxActionPerformed"/>
</Events>
</Component>
</SubComponents>
</Form>

23
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchModuleSettingsPanel.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
@ -38,7 +38,6 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett
private void customizeComponents() {
skipNoExtCheckBox.setSelected(settings.skipFilesWithNoExtension());
skipTextPlain.setSelected(settings.skipFilesWithTextPlainMimeType());
skipKnownFilesCheckbox.setSelected(settings.skipKnownFiles());
}
@Override
@ -57,7 +56,6 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett
skipNoExtCheckBox = new javax.swing.JCheckBox();
skipTextPlain = new javax.swing.JCheckBox();
skipKnownFilesCheckbox = new javax.swing.JCheckBox();
skipNoExtCheckBox.setSelected(true);
skipNoExtCheckBox.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchModuleSettingsPanel.class, "FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text")); // NOI18N
@ -75,13 +73,6 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett
}
});
skipKnownFilesCheckbox.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchModuleSettingsPanel.class, "FileExtMismatchModuleSettingsPanel.skipKnownFilesCheckbox.text")); // NOI18N
skipKnownFilesCheckbox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
skipKnownFilesCheckboxActionPerformed(evt);
}
});
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
@ -90,8 +81,7 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(skipTextPlain)
.addComponent(skipNoExtCheckBox)
.addComponent(skipKnownFilesCheckbox))
.addComponent(skipNoExtCheckBox))
.addGap(0, 138, Short.MAX_VALUE))
);
layout.setVerticalGroup(
@ -100,9 +90,7 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett
.addComponent(skipNoExtCheckBox)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(skipTextPlain)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(skipKnownFilesCheckbox)
.addContainerGap(28, Short.MAX_VALUE))
.addContainerGap(51, Short.MAX_VALUE))
);
}// </editor-fold>//GEN-END:initComponents
@ -114,12 +102,7 @@ final class FileExtMismatchModuleSettingsPanel extends IngestModuleIngestJobSett
settings.setSkipFilesWithTextPlainMimeType(skipTextPlain.isSelected());
}//GEN-LAST:event_skipTextPlainActionPerformed
private void skipKnownFilesCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipKnownFilesCheckboxActionPerformed
settings.setSkipKnownFiles(skipKnownFilesCheckbox.isSelected());
}//GEN-LAST:event_skipKnownFilesCheckboxActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JCheckBox skipKnownFilesCheckbox;
private javax.swing.JCheckBox skipNoExtCheckBox;
private javax.swing.JCheckBox skipTextPlain;
// End of variables declaration//GEN-END:variables

4
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchOptionsPanelController.java
View File

@ -2,7 +2,7 @@
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
@ -17,7 +17,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
@OptionsPanelController.TopLevelRegistration(
categoryName = "#OptionsCategory_Name_FileExtMismatchOptions",
iconBase = "org/sleuthkit/autopsy/fileextmismatch/options-icon.png",
iconBase = "org/sleuthkit/autopsy/modules/fileextmismatch/options-icon.png",
position = 4,
keywords = "#OptionsCategory_FileExtMismatch",
keywordsCategory = "KeywordSearchOptions")

30
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.form → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.form
View File

@ -50,10 +50,10 @@
<Component class="javax.swing.JButton" name="saveButton">
<Properties>
<Property name="icon" type="javax.swing.Icon" editor="org.netbeans.modules.form.editors2.IconEditor">
<Image iconType="3" name="/org/sleuthkit/autopsy/fileextmismatch/save16.png"/>
<Image iconType="3" name="/org/sleuthkit/autopsy/modules/fileextmismatch/save16.png"/>
</Property>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.saveButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.saveButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="enabled" type="boolean" value="false"/>
</Properties>
@ -128,7 +128,7 @@
<Component class="javax.swing.JLabel" name="jLabel1">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.jLabel1.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.jLabel1.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
@ -148,7 +148,7 @@
<Component class="javax.swing.JTextField" name="userTypeTextField">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.userTypeTextField.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.userTypeTextField.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -158,7 +158,7 @@
<Component class="javax.swing.JButton" name="addTypeButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.addTypeButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.addTypeButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -168,7 +168,7 @@
<Component class="javax.swing.JButton" name="removeTypeButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.removeTypeButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.removeTypeButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -181,14 +181,14 @@
<Color blue="0" green="0" red="ff" type="rgb"/>
</Property>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.mimeErrLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.mimeErrLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JLabel" name="mimeRemoveErrLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.mimeRemoveErrLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.mimeRemoveErrLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
@ -254,7 +254,7 @@
<Component class="javax.swing.JTextField" name="userExtTextField">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.userExtTextField.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.userExtTextField.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -264,7 +264,7 @@
<Component class="javax.swing.JButton" name="addExtButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.addExtButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.addExtButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -287,7 +287,7 @@
<Component class="javax.swing.JButton" name="removeExtButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.removeExtButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.removeExtButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
@ -297,7 +297,7 @@
<Component class="javax.swing.JLabel" name="extHeaderLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.extHeaderLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.extHeaderLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
@ -307,14 +307,14 @@
<Color blue="0" green="0" red="ff" type="rgb"/>
</Property>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.extErrorLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.extErrorLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JLabel" name="extRemoveErrLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.extRemoveErrLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.extRemoveErrLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
@ -328,7 +328,7 @@
<Color blue="ff" green="0" red="0" type="rgb"/>
</Property>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.saveMsgLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties" key="FileExtMismatchSettingsPanel.saveMsgLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>

6
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchSettingsPanel.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import java.awt.Color;
import java.util.ArrayList;
@ -32,7 +32,7 @@ import javax.swing.table.AbstractTableModel;
import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.filetypeid.FileTypeIdIngestModule;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdIngestModule;
import org.sleuthkit.autopsy.corecomponents.OptionsPanel;
/**
@ -154,7 +154,7 @@ final class FileExtMismatchSettingsPanel extends IngestModuleGlobalSetttingsPane
extRemoveErrLabel = new javax.swing.JLabel();
saveMsgLabel = new javax.swing.JLabel();
saveButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/fileextmismatch/save16.png"))); // NOI18N
saveButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/fileextmismatch/save16.png"))); // NOI18N
saveButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.saveButton.text")); // NOI18N
saveButton.setEnabled(false);
saveButton.addActionListener(new java.awt.event.ActionListener() {

4
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchXML.java → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchXML.java
View File

@ -17,7 +17,7 @@
* limitations under the License.
*/
package org.sleuthkit.autopsy.fileextmismatch;
package org.sleuthkit.autopsy.modules.fileextmismatch;
import java.io.File;
import java.io.IOException;
@ -61,7 +61,7 @@ class FileExtMismatchXML {
this.filePath = filePath;
try {
boolean extracted = PlatformUtil.extractResourceToUserConfigDir(FileExtMismatchXML.class, DEFAULT_CONFIG_FILE_NAME);
boolean extracted = PlatformUtil.extractResourceToUserConfigDir(FileExtMismatchXML.class, DEFAULT_CONFIG_FILE_NAME, false);
} catch (IOException ex) {
logger.log(Level.SEVERE, "Error copying default mismatch configuration to user dir ", ex);
}

0
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/MismatchConfigSchema.xsd → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/MismatchConfigSchema.xsd
View File

0
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/mismatch_config.xml → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/mismatch_config.xml
View File

0
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/options-icon.png → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/options-icon.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

0
FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/save16.png → Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/save16.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 609 B

After

Width:  |  Height:  |  Size: 609 B

0
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle.properties → Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties
View File

0
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle_ja.properties → Core/src/org/sleuthkit/autopsy/modules/filetypeid/Bundle_ja.properties
View File

2
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java → Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetectionInterface.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.filetypeid;
package org.sleuthkit.autopsy.modules.filetypeid;
import org.sleuthkit.datamodel.AbstractFile;

57
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java → Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java
View File

@ -16,13 +16,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.filetypeid;
package org.sleuthkit.autopsy.modules.filetypeid;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.datamodel.AbstractFile;
@ -34,6 +35,7 @@ import org.sleuthkit.datamodel.TskData.FileKnown;
import org.sleuthkit.datamodel.TskException;
import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
/**
* Detects the type of a file based on signature (magic) values. Posts results
@ -44,9 +46,11 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName());
private static final long MIN_FILE_SIZE = 512;
private final FileTypeIdModuleSettings settings;
private long matchTime = 0;
private int messageId = 0; // RJCTODO: If this is not made a thread safe static, duplicate message ids will be used
private long numFiles = 0;
private long jobId;
private static AtomicLong matchTime = new AtomicLong(0);
private static AtomicLong numFiles = new AtomicLong(0);
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
// The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed.
// If desired in the future to be more knowledgable about weird files or rare formats, we could
// actually have a list of detectors which are called in order until a match is found.
@ -56,6 +60,12 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
this.settings = settings;
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
jobId = context.getJobId();
refCounter.incrementAndGet(jobId);
}
@Override
public ProcessResult process(AbstractFile abstractFile) {
// skip non-files
@ -76,8 +86,8 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
try {
long startTime = System.currentTimeMillis();
FileTypeDetectionInterface.FileIdInfo fileId = detector.attemptMatch(abstractFile);
matchTime += (System.currentTimeMillis() - startTime);
numFiles++;
matchTime.getAndAdd(System.currentTimeMillis() - startTime);
numFiles.getAndIncrement();
if (!fileId.type.isEmpty()) {
// add artifact
@ -99,22 +109,25 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
@Override
public void shutDown(boolean ingestJobCancelled) {
StringBuilder detailsSb = new StringBuilder();
detailsSb.append("<table border='0' cellpadding='4' width='280'>");
detailsSb.append("<tr><td>").append(FileTypeIdModuleFactory.getModuleName()).append("</td></tr>");
detailsSb.append("<tr><td>")
.append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime"))
.append("</td><td>").append(matchTime).append("</td></tr>\n");
detailsSb.append("<tr><td>")
.append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles"))
.append("</td><td>").append(numFiles).append("</td></tr>\n");
detailsSb.append("</table>");
IngestServices.getInstance().postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"FileTypeIdIngestModule.complete.srvMsg.text"),
detailsSb.toString()));
// We only need to post the summary msg from the last module per job
if (refCounter.decrementAndGet(jobId) == 0) {
StringBuilder detailsSb = new StringBuilder();
detailsSb.append("<table border='0' cellpadding='4' width='280'>");
detailsSb.append("<tr><td>").append(FileTypeIdModuleFactory.getModuleName()).append("</td></tr>");
detailsSb.append("<tr><td>")
.append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime"))
.append("</td><td>").append(matchTime.get()).append("</td></tr>\n");
detailsSb.append("<tr><td>")
.append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles"))
.append("</td><td>").append(numFiles.get()).append("</td></tr>\n");
detailsSb.append("</table>");
IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"FileTypeIdIngestModule.complete.srvMsg.text"),
detailsSb.toString()));
}
}
/**
* Validate if a given mime type is in the detector's registry.
*

2
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleFactory.java → Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleFactory.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.filetypeid;
package org.sleuthkit.autopsy.modules.filetypeid;
import org.openide.util.NbBundle;
import org.openide.util.lookup.ServiceProvider;

2
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettings.java → Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettings.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.filetypeid;
package org.sleuthkit.autopsy.modules.filetypeid;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;

6
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.form → Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.form
View File

@ -38,13 +38,13 @@
<Properties>
<Property name="selected" type="boolean" value="true"/>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/filetypeid/Bundle.properties" key="FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties" key="FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/filetypeid/Bundle.properties" key="FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties" key="FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="label" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/filetypeid/Bundle.properties" key="FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/modules/filetypeid/Bundle.properties" key="FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>

2
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.java → Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdModuleSettingsPanel.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.filetypeid;
package org.sleuthkit.autopsy.modules.filetypeid;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;

7
FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java → Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.filetypeid;
package org.sleuthkit.autopsy.modules.filetypeid;
import java.util.SortedSet;
import org.apache.tika.Tika;
@ -35,7 +35,8 @@ class TikaFileTypeDetector implements FileTypeDetectionInterface {
FileTypeDetectionInterface.FileIdInfo ret = new FileTypeDetectionInterface.FileIdInfo();
final int maxBytesInitial = 100; //how many bytes to read on first pass
byte buffer[] = new byte[maxBytesInitial];
int len = abstractFile.read(buffer, 0, maxBytesInitial);
boolean found = false;
try {
// the xml detection in Tika tries to parse the entire file and throws exceptions
@ -49,7 +50,7 @@ class TikaFileTypeDetector implements FileTypeDetectionInterface {
}
catch (IndexOutOfBoundsException e) {
// do nothing
}
}
if (found == false) {
String mimetype = tikaInst.detect(buffer);

2
SevenZip/src/org/sleuthkit/autopsy/sevenzip/ArchiveFileExtractorModuleFactory.java → Core/src/org/sleuthkit/autopsy/modules/sevenzip/ArchiveFileExtractorModuleFactory.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.sevenzip;
package org.sleuthkit.autopsy.modules.sevenzip;
import org.openide.util.NbBundle;
import org.openide.util.lookup.ServiceProvider;

0
SevenZip/src/org/sleuthkit/autopsy/sevenzip/Bundle.properties → Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle.properties
View File

0
SevenZip/src/org/sleuthkit/autopsy/sevenzip/Bundle_ja.properties → Core/src/org/sleuthkit/autopsy/modules/sevenzip/Bundle_ja.properties
View File

2
SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipContentReadStream.java → Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipContentReadStream.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.sevenzip;
package org.sleuthkit.autopsy.modules.sevenzip;
import java.io.IOException;
import java.util.logging.Level;

54
SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipIngestModule.java → Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.sevenzip;
package org.sleuthkit.autopsy.modules.sevenzip;
import java.io.BufferedOutputStream;
import java.io.File;
@ -60,6 +60,7 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
/**
* 7Zip ingest module extracts supported archives, adds extracted DerivedFiles,
@ -69,7 +70,6 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
private static final Logger logger = Logger.getLogger(SevenZipIngestModule.class.getName());
private IngestServices services = IngestServices.getInstance();
private volatile int messageID = 0; // RJCTODO: This is not actually thread safe
static final String[] SUPPORTED_EXTENSIONS = {"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",}; // "iso"};
private String unpackDir; //relative to the case, to store in db
private String unpackDirPath; //absolute, to extract to
@ -91,6 +91,8 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
private final byte[] fileHeaderBuffer = new byte[readHeaderSize];
private static final int ZIP_SIGNATURE_BE = 0x504B0304;
private IngestJobContext context;
private long jobId;
private final static IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
SevenZipIngestModule() {
}
@ -98,6 +100,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context;
jobId = context.getJobId();
final Case currentCase = Case.getCurrentCase();
@ -117,25 +120,28 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String details = NbBundle.getMessage(this.getClass(),
"SevenZipIngestModule.init.errInitModule.details",
unpackDirPath, e.getMessage());
services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
throw e;
}
}
try {
SevenZip.initSevenZipFromPlatformJAR();
String platform = SevenZip.getUsedPlatform();
logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform);
} catch (SevenZipNativeInitializationException e) {
logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e);
String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg",
ArchiveFileExtractorModuleFactory.getModuleName());
String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib",
e.getMessage());
services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
throw new RuntimeException(e);
// if first instance of this module for this job then check 7zip init
if (refCounter.incrementAndGet(jobId) == 1) {
try {
SevenZip.initSevenZipFromPlatformJAR();
String platform = SevenZip.getUsedPlatform();
logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform);
} catch (SevenZipNativeInitializationException e) {
logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e);
String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg",
ArchiveFileExtractorModuleFactory.getModuleName());
String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib",
e.getMessage());
services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
throw new RuntimeException(e);
}
}
archiveDepthCountTree = new ArchiveDepthCountTree();
}
@ -180,6 +186,12 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
return ProcessResult.OK;
}
@Override
public void shutDown(boolean ingestJobCancelled) {
// We don't need the value, but for cleanliness and consistency, -- it
refCounter.decrementAndGet(jobId);
}
private void sendNewFilesEvent(AbstractFile archive, List<AbstractFile> unpackedFiles) {
//currently sending a single event for all new files
services.fireModuleContentEvent(new ModuleContentEvent(archive));
@ -242,7 +254,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String details = NbBundle.getMessage(this.getClass(),
"SevenZipIngestModule.isZipBombCheck.warnDetails", cRatio);
//MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(IngestMessage.createWarningMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
services.postMessage(IngestMessage.createWarningMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
return true;
} else {
return false;
@ -276,7 +288,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
"SevenZipIngestModule.unpack.warnDetails.zipBomb",
parentAr.getDepth());
//MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(IngestMessage.createWarningMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
services.postMessage(IngestMessage.createWarningMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
return unpackedFiles;
}
@ -403,7 +415,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String details = NbBundle.getMessage(this.getClass(),
"SevenZipIngestModule.unpack.notEnoughDiskSpace.details");
//MessageNotifyUtil.Notify.error(msg, details);
services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
logger.log(Level.INFO, "Skipping archive item due not sufficient disk space for this item: {0}, {1}", new Object[]{archiveFile.getName(), fileName});
continue; //skip this file
} else {
@ -501,7 +513,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String details = NbBundle.getMessage(this.getClass(),
"SevenZipIngestModule.unpack.errUnpacking.details",
fullName, ex.getMessage());
services.postMessage(IngestMessage.createErrorMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
}
} finally {
if (inArchive != null) {
@ -541,7 +553,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String details = NbBundle.getMessage(this.getClass(),
"SevenZipIngestModule.unpack.encrFileDetected.details",
archiveFile.getName(), ArchiveFileExtractorModuleFactory.getModuleName());
services.postMessage(IngestMessage.createWarningMessage(++messageID, ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
services.postMessage(IngestMessage.createWarningMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
}
return unpackedFiles;

7
Core/src/org/sleuthkit/autopsy/report/Bundle.properties
View File

@ -47,6 +47,13 @@ ReportBodyFile.progress.processing=Now processing {0}...
ReportBodyFile.getName.text=TSK Body File
ReportBodyFile.getDesc.text=Body file format report with MAC times for every file. This format can be used for a timeline view.
ReportBodyFile.getFilePath.text=BodyFile.txt
ReportKML.progress.querying=Querying files...
ReportKML.ingestWarning.text=Warning, this report was run before ingest services completed\!
ReportKML.progress.loading=Loading files...
ReportKML.progress.processing=Now processing {0}...
ReportKML.getName.text=Google Earth/KML
ReportKML.getDesc.text=KML format report with coordinates for relevant files. This format can be used for google earth views.
ReportKML.getFilePath.text=ReportKML.kml
ReportBranding.defaultReportTitle.text=Autopsy Forensic Report
ReportBranding.defaultReportFooter.text=Powered by Autopsy Open Source Digital Forensics Platform - www.sleuthkit.org
ReportExcel.numAartifacts.text=Number of artifacts\:

318
Core/src/org/sleuthkit/autopsy/report/ReportKML.java Normal file
View File

@ -0,0 +1,318 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.report;
import javax.swing.JPanel;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.*;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.datamodel.BlackboardArtifact;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.logging.Level;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.jdom2.output.Format;
import org.jdom2.output.XMLOutputter;
import org.apache.commons.lang.StringEscapeUtils;
/**
* Generates a KML file based on geo coordinates store in blackboard.
*/
class ReportKML implements GeneralReportModule {
private static final Logger logger = Logger.getLogger(ReportKML.class.getName());
private static ReportKML instance = null;
private Case currentCase;
private SleuthkitCase skCase;
private String reportPath;
// Hidden constructor for the report
private ReportKML() {
}
// Get the default implementation of this report
public static synchronized ReportKML getDefault() {
if (instance == null) {
instance = new ReportKML();
}
return instance;
}
/**
* Generates a body file format report for use with the MAC time tool.
*
* @param path path to save the report
* @param progressPanel panel to update the report's progress
*/
@Override
public void generateReport(String path, ReportProgressPanel progressPanel) {
// Start the progress bar and setup the report
progressPanel.setIndeterminate(false);
progressPanel.start();
progressPanel.updateStatusLabel(NbBundle.getMessage(this.getClass(), "ReportKML.progress.querying"));
reportPath = path + "ReportKML.kml";
String reportPath2 = path + "ReportKML.txt";
currentCase = Case.getCurrentCase();
skCase = currentCase.getSleuthkitCase();
progressPanel.updateStatusLabel(NbBundle.getMessage(this.getClass(), "ReportKML.progress.loading"));
// Check if ingest has finished
String ingestwarning = "";
if (IngestManager.getInstance().isIngestRunning()) {
ingestwarning = NbBundle.getMessage(this.getClass(), "ReportBodyFile.ingestWarning.text");
}
progressPanel.setMaximumProgress(5);
progressPanel.increment();
try {
BufferedWriter out = null;
try {
out = new BufferedWriter(new FileWriter(reportPath2));
double lat = 0; // temp latitude
double lon = 0; //temp longitude
AbstractFile aFile;
String geoPath = ""; // will hold values of images to put in kml
String imageName = "";
File f;
for (BlackboardArtifact artifact : skCase.getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)) {
lat = 0;
lon = 0;
geoPath = "";
String extractedToPath;
for (BlackboardAttribute attribute : artifact.getAttributes()) {
if (attribute.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID()) //latitude
{
lat = attribute.getValueDouble();
}
if (attribute.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID()) //longitude
{
lon = attribute.getValueDouble();
}
}
if (lon != 0 && lat != 0) {
aFile = artifact.getSleuthkitCase().getAbstractFileById(artifact.getObjectID());
extractedToPath = reportPath + aFile.getName();
geoPath = extractedToPath;
f = new File(extractedToPath);
f.createNewFile();
copyFileUsingStream(aFile, f);
imageName = aFile.getName();
out.write(String.valueOf(lat));
out.write(";");
out.write(String.valueOf(lon));
out.write(";");
out.write(String.valueOf(geoPath));
out.write(";");
out.write(String.valueOf(imageName));
out.write("\n");
// lat lon path name
}
}
out.flush();
out.close();
progressPanel.increment();
/*
* Step 1: generate XML stub
*/
Namespace ns = Namespace.getNamespace("", "http://earth.google.com/kml/2.2");
// kml
Element kml = new Element("kml", ns);
Document kmlDocument = new Document(kml);
// Document
Element document = new Element("Document", ns);
kml.addContent(document);
// name
Element name = new Element("name", ns);
name.setText("Java Generated KML Document");
document.addContent(name);
/*
* Step 2: add in Style elements
*/
// Style
Element style = new Element("Style", ns);
style.setAttribute("id", "redIcon");
document.addContent(style);
// IconStyle
Element iconStyle = new Element("IconStyle", ns);
style.addContent(iconStyle);
// color
Element color = new Element("color", ns);
color.setText("990000ff");
iconStyle.addContent(color);
// Icon
Element icon = new Element("Icon", ns);
iconStyle.addContent(icon);
// href
Element href = new Element("href", ns);
href.setText("http://www.cs.mun.ca/~hoeber/teaching/cs4767/notes/02.1-kml/circle.png");
icon.addContent(href);
progressPanel.increment();
/*
* Step 3: read data from source location and
* add in a Placemark for each data element
*/
File file = new File(reportPath2);
BufferedReader reader;
reader = new BufferedReader(new FileReader(file));
String line = reader.readLine();
while (line != null) {
String[] lineParts = line.split(";");
if (lineParts.length == 4) {
String coordinates = lineParts[1].trim() + "," + lineParts[0].trim(); //lat,lon
// Placemark
Element placemark = new Element("Placemark", ns);
document.addContent(placemark);
// name
Element pmName = new Element("name", ns);
pmName.setText(lineParts[3].trim());
placemark.addContent(pmName);
// Path
Element pmPath = new Element("Path", ns);
pmPath.setText(lineParts[2].trim());
placemark.addContent(pmPath);
// description
Element pmDescription = new Element("description", ns);
String xml = "<![CDATA[ \n" + " <img src='file:///" + lineParts[2].trim() + "' width='400' /><br/&gt; \n";
StringEscapeUtils.unescapeXml(xml);
pmDescription.setText(xml);
placemark.addContent(pmDescription);
// styleUrl
Element pmStyleUrl = new Element("styleUrl", ns);
pmStyleUrl.setText("#redIcon");
placemark.addContent(pmStyleUrl);
// Point
Element pmPoint = new Element("Point", ns);
placemark.addContent(pmPoint);
// coordinates
Element pmCoordinates = new Element("coordinates", ns);
pmCoordinates.setText(coordinates);
pmPoint.addContent(pmCoordinates);
}
// read the next line
line = reader.readLine();
}
progressPanel.increment();
/*
* Step 4: write the XML file
*/
try {
XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat());
FileOutputStream writer = new FileOutputStream(reportPath);
outputter.output(kmlDocument, writer);
writer.close();
} catch (IOException ex) {
logger.log(Level.WARNING, "Could not write the KML file.", ex);
}
} catch (IOException ex) {
logger.log(Level.WARNING, "Could not write the KML report.", ex);
}
progressPanel.complete();
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Failed to get the unique path.", ex);
}
progressPanel.increment();
progressPanel.complete();
}
public static void copyFileUsingStream(AbstractFile file, File jFile) throws IOException {
InputStream is = new ReadContentInputStream(file);
OutputStream os = new FileOutputStream(jFile);
byte[] buffer = new byte[8192];
int length;
try {
while ((length = is.read(buffer)) != -1) {
os.write(buffer, 0, length);
}
} finally {
is.close();
os.close();
}
}
@Override
public String getName() {
String name = NbBundle.getMessage(this.getClass(), "ReportKML.getName.text");
return name;
}
@Override
public String getFilePath() {
return NbBundle.getMessage(this.getClass(), "ReportKML.getFilePath.text");
}
@Override
public String getExtension() {
String ext = ".txt";
return ext;
}
@Override
public String getDescription() {
String desc = NbBundle.getMessage(this.getClass(), "ReportKML.getDesc.text");
return desc;
}
@Override
public JPanel getConfigurationPanel() {
return null; // No configuration panel
}
}

0
Timeline/src/org/sleuthkit/autopsy/timeline/Bundle.properties → Core/src/org/sleuthkit/autopsy/timeline/Bundle.properties
View File

0
Timeline/src/org/sleuthkit/autopsy/timeline/Bundle_ja.properties → Core/src/org/sleuthkit/autopsy/timeline/Bundle_ja.properties
View File

0
Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java → Core/src/org/sleuthkit/autopsy/timeline/Timeline.java
View File

0
Timeline/src/org/sleuthkit/autopsy/timeline/TimelineFrame.form → Core/src/org/sleuthkit/autopsy/timeline/TimelineFrame.form
View File

0
Timeline/src/org/sleuthkit/autopsy/timeline/TimelineFrame.java → Core/src/org/sleuthkit/autopsy/timeline/TimelineFrame.java
View File

0
Timeline/src/org/sleuthkit/autopsy/timeline/TimelineProgressDialog.form → Core/src/org/sleuthkit/autopsy/timeline/TimelineProgressDialog.form
View File

0
Timeline/src/org/sleuthkit/autopsy/timeline/TimelineProgressDialog.java → Core/src/org/sleuthkit/autopsy/timeline/TimelineProgressDialog.java
View File

Some files were not shown because too many files have changed in this diff Show More