From 938b051f4843496e532b4b8b9ea151ee62b3556a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\zhaohui" Date: Thu, 28 Sep 2017 11:58:33 -0400 Subject: [PATCH 01/30] 3095: Expanding Data Sources Node in nightly test --- .../autopsy/testing/AutopsyTestCases.java | 24 +++++++++++++++++++ .../autopsy/testing/RegressionTest.java | 5 ++++ 2 files changed, 29 insertions(+) diff --git a/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java b/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java index 672ca5b22b..c716081672 100755 --- a/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java +++ b/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java @@ -36,6 +36,7 @@ import java.util.logging.Logger; import javax.imageio.ImageIO; import javax.swing.JDialog; import javax.swing.text.JTextComponent; +import javax.swing.tree.TreePath; import org.netbeans.jellytools.MainWindowOperator; import org.netbeans.jellytools.NbDialogOperator; import org.netbeans.jellytools.WizardOperator; @@ -53,6 +54,8 @@ import org.netbeans.jemmy.operators.JTabbedPaneOperator; import org.netbeans.jemmy.operators.JTableOperator; import org.netbeans.jemmy.operators.JTextFieldOperator; import org.netbeans.jemmy.operators.JToggleButtonOperator; +import org.netbeans.jemmy.operators.JTreeOperator; +import org.netbeans.jemmy.operators.JTreeOperator.NoSuchPathException; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.core.UserPreferencesException; import org.sleuthkit.autopsy.events.MessageServiceConnectionInfo; @@ -290,6 +293,16 @@ public class AutopsyTestCases { } + public void testExpandDataSourcesTree() { + logger.info("Data Sources Node"); + MainWindowOperator mwo = MainWindowOperator.getDefault(); + JTreeOperator jto = new JTreeOperator(mwo, "Data Sources"); + String [] nodeNames = {"Data Sources"}; + TreePath tp = jto.findPath(nodeNames); + expandNodes(jto, tp); + screenshot("Data Sources Tree"); + } + public void testGenerateReportToolbar() { logger.info("Generate Report Toolbars"); MainWindowOperator mwo = MainWindowOperator.getDefault(); @@ -380,4 +393,15 @@ public class AutopsyTestCases { logger.log(Level.SEVERE, "Error saving messaging service connection info", ex); //NON-NLS } } + + private void expandNodes (JTreeOperator jto, TreePath tp) { + try { + jto.expandPath(tp); + for (TreePath t : jto.getChildPaths(tp)) { + expandNodes(jto, t); + } + } catch (NoSuchPathException ne) { + logger.log(Level.SEVERE, "Error expanding tree path", ne); + } + } } diff --git a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java index 674395e0f1..0518da865b 100755 --- a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java +++ b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java @@ -69,6 +69,7 @@ public class RegressionTest extends TestCase { "testConfigureSearch", "testAddSourceWizard1", "testIngest", + "testExpandDataSourcesTree", //After do ingest, before generate report, we expand Data Sources node "testGenerateReportToolbar", "testGenerateReportButton"); } @@ -83,6 +84,7 @@ public class RegressionTest extends TestCase { "testConfigureSearch", "testAddSourceWizard1", "testIngest", + "testExpandDataSourcesTree", "testGenerateReportToolbar", "testGenerateReportButton"); } @@ -147,6 +149,9 @@ public class RegressionTest extends TestCase { autopsyTests.testIngest(); } + public void testExpandDataSourcesTree() { + autopsyTests.testExpandDataSourcesTree(); + } public void testGenerateReportToolbar() { autopsyTests.testGenerateReportToolbar(); } From 887e4d8f06d516cf16e59c87b573e934b265ba91 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 2 Oct 2017 11:47:42 -0400 Subject: [PATCH 02/30] 3039 change tagged content color from dark gray to light yellow --- .../sleuthkit/autopsy/corecomponents/DataResultViewerTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 25cd62e719..e922e74c4f 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -79,7 +79,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { private static final Logger logger = Logger.getLogger(DataResultViewerTable.class.getName()); @NbBundle.Messages("DataResultViewerTable.firstColLbl=Name") static private final String FIRST_COLUMN_LABEL = Bundle.DataResultViewerTable_firstColLbl(); - private static final Color TAGGED_COLOR = new Color(200, 210, 220); + private static final Color TAGGED_COLOR = new Color(255, 255, 195); /** * The properties map: * From db0ff82c731ae6c3533682a64a76bc61252ca674 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 3 Oct 2017 15:53:12 -0400 Subject: [PATCH 03/30] Restore organizations panel in EamCaseEditDetailsDialog --- .../centralrepository/actions/EamCaseEditDetailsDialog.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/actions/EamCaseEditDetailsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/actions/EamCaseEditDetailsDialog.java index d647c12d9b..3e34bf9dd2 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/actions/EamCaseEditDetailsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/actions/EamCaseEditDetailsDialog.java @@ -85,10 +85,6 @@ public class EamCaseEditDetailsDialog extends JDialog { private void customizeComponents() { setTextBoxListeners(); setTextAreaListeners(); - - // The organization functions of central repo are not being included in the current release. - this.pnOrganization.setVisible(false); - } private void setTextBoxListeners() { From 5eefb7abda74236227e76c8191bc2e3736cec964 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 3 Oct 2017 19:59:30 -0400 Subject: [PATCH 04/30] Change progress bar msg of ImageWriter --- .../src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java index 084c716d5a..3cd166a1b5 100755 --- a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java +++ b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.ExecutionException; import java.util.logging.Level; import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; @@ -130,6 +131,10 @@ class ImageWriter implements PropertyChangeListener{ } } + @Messages({ + "# {0} - data source name", + "ImageWriter.progressBar.message=Finishing acquisition of {0}" + }) private void startFinishImage(String dataSourceName){ synchronized(currentTasksLock){ @@ -166,7 +171,7 @@ class ImageWriter implements PropertyChangeListener{ if(doUI){ periodicTasksExecutor = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().setNameFormat("image-writer-progress-update-%d").build()); //NON-NLS - progressHandle = ProgressHandle.createHandle("Image writer - " + dataSourceName); + progressHandle = ProgressHandle.createHandle(Bundle.ImageWriter_progressBar_message(dataSourceName)); progressHandle.start(100); progressUpdateTask = periodicTasksExecutor.scheduleAtFixedRate( new ProgressUpdateTask(progressHandle, imageHandle), 0, 250, TimeUnit.MILLISECONDS); From cf6312cdb3d5e3440109b37993c1e6f54c08629c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 5 Oct 2017 11:15:37 -0400 Subject: [PATCH 05/30] Reworked 'Examiner' mode so it works without restart. --- .../AutoIngestCasePanelInterface.java | 29 +++ .../autopsy/casemodule/Bundle.properties | 14 +- .../autopsy/casemodule/Bundle_ja.properties | 7 +- .../autopsy/casemodule/CueBannerPanel.form | 128 +++++++---- .../autopsy/casemodule/CueBannerPanel.java | 200 +++++++++++------- .../autoingest/AutoIngestCaseManager.java | 13 -- .../autoingest/AutoIngestCasePanel.java | 42 +++- .../AutoIngestSettingsPanel.form | 21 +- .../AutoIngestSettingsPanel.java | 52 +---- .../AutoIngestUserPreferences.java | 4 +- .../configuration/Bundle.properties | 8 +- .../configuration/StartupWindow.java | 62 ++---- 12 files changed, 331 insertions(+), 249 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/casemodule/AutoIngestCasePanelInterface.java diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AutoIngestCasePanelInterface.java b/Core/src/org/sleuthkit/autopsy/casemodule/AutoIngestCasePanelInterface.java new file mode 100755 index 0000000000..795b636b35 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AutoIngestCasePanelInterface.java @@ -0,0 +1,29 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.casemodule; + +import javax.swing.JDialog; + +/** + * Interface for startup window implementations + */ +public interface AutoIngestCasePanelInterface { + + public void addWindowStateListener(JDialog parent); +} diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties index 8ba843a41c..1fc2209750 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties @@ -17,13 +17,7 @@ NewCaseVisualPanel1.jLabel2.text_1=Case data will be stored in the following dir NewCaseVisualPanel1.caseParentDirTextField.text= NewCaseVisualPanel1.caseDirTextField.text_1= CueBannerPanel.autopsyLogo.text= -CueBannerPanel.createNewLabel.text=Create New Case -CueBannerPanel.openLabel.text=Open Existing Case CueBannerPanel.closeButton.text=Close -CueBannerPanel.openRecentLabel.text=Open Recent Case -CueBannerPanel.newCaseButton.text= -CueBannerPanel.openCaseButton.text= -CueBannerPanel.openRecentButton.text= OpenRecentCasePanel.cancelButton.text=Cancel OpenRecentCasePanel.jLabel1.text=Recent Cases NewCaseVisualPanel2.caseNumberTextField.text= @@ -225,3 +219,11 @@ CasePropertiesPanel.lbDbType.text=Case Type: CasePropertiesPanel.examinerLabel.text=Examiner: CasePropertiesPanel.caseNumberLabel.text=Case Number: LocalDiskPanel.changeDatabasePathCheckbox.text=Update case to use VHD file upon completion +CueBannerPanel.openAutoIngestCaseButton.text= +CueBannerPanel.openExistingCaseButton.text= +CueBannerPanel.openRecentCaseButton.text= +CueBannerPanel.createNewCaseButton.text= +CueBannerPanel.createNewCaseLabel.text=Create New Case +CueBannerPanel.openRecentCaseLabel.text=Open Recent Case +CueBannerPanel.openExistingCaseLabel.text=Open Existing Case +CueBannerPanel.openAutoIngestCaseLabel.text=Open Auto Ingest Case diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties index 939c901e57..fd00f725e5 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle_ja.properties @@ -12,10 +12,7 @@ NewCaseVisualPanel1.caseNameLabel.text_1=\u30b1\u30fc\u30b9\u540d\uff1a NewCaseVisualPanel1.caseDirLabel.text=\u30d9\u30fc\u30b9\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\uff1a NewCaseVisualPanel1.caseDirBrowseButton.text=\u95b2\u89a7 NewCaseVisualPanel1.jLabel2.text_1=\u30b1\u30fc\u30b9\u30c7\u30fc\u30bf\u306f\u6b21\u306e\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u306b\u4fdd\u5b58\u3055\u308c\u307e\u3059\uff1a -CueBannerPanel.createNewLabel.text=\u65b0\u898f\u30b1\u30fc\u30b9\u3092\u4f5c\u6210 -CueBannerPanel.openLabel.text=\u65e2\u5b58\u30b1\u30fc\u30b9\u3092\u958b\u304f CueBannerPanel.closeButton.text=\u9589\u3058\u308b -CueBannerPanel.openRecentLabel.text=\u6700\u8fd1\u958b\u3044\u305f\u30b1\u30fc\u30b9\u3092\u958b\u304f OpenRecentCasePanel.cancelButton.text=\u30ad\u30e3\u30f3\u30bb\u30eb OpenRecentCasePanel.jLabel1.text=\u6700\u8fd1\u958b\u3044\u305f\u30d5\u30a1\u30a4\u30eb NewCaseVisualPanel2.examinerLabel.text=\u8abf\u67fb\u62c5\u5f53\u8005\uff1a @@ -196,3 +193,7 @@ CasePropertiesPanel.lbDbName.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u540d\uff CasePropertiesPanel.lbDbType.text=\u30b1\u30fc\u30b9\u30bf\u30a4\u30d7\uff1a CasePropertiesPanel.examinerLabel.text=\u8abf\u67fb\u62c5\u5f53\u8005\uff1a CasePropertiesPanel.caseNumberLabel.text=\u30b1\u30fc\u30b9\u756a\u53f7\uff1a +CueBannerPanel.createNewCaseLabel.text=\u65b0\u898f\u30b1\u30fc\u30b9\u3092\u4f5c\u6210 +CueBannerPanel.openRecentCaseLabel.text=\u6700\u8fd1\u958b\u3044\u305f\u30b1\u30fc\u30b9\u3092\u958b\u304f +CueBannerPanel.openExistingCaseLabel.text=\u65e2\u5b58\u30b1\u30fc\u30b9\u3092\u958b\u304f +CueBannerPanel.openAutoIngestCaseLabel.text=\u65e2\u5b58\u30b1\u30fc\u30b9\u3092\u958b\u304f diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.form b/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.form index 223b83b7d4..75819e41d0 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.form +++ b/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.form @@ -16,23 +16,27 @@ - - - - - - - - + @@ -159,10 +155,7 @@ - - - - + @@ -203,16 +196,6 @@ - - - - - - - - - - diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java index d64b9954ab..4877ce9d60 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestSettingsPanel.java @@ -210,35 +210,16 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { boolean needsRestart = false; UserPreferences.SelectedMode storedMode = UserPreferences.getMode(); - if (AutoIngestUserPreferences.getJoinAutoModeCluster() != cbJoinAutoIngestCluster.isSelected()) { - needsRestart = true; - } - AutoIngestUserPreferences.setJoinAutoModeCluster(cbJoinAutoIngestCluster.isSelected()); if (!cbJoinAutoIngestCluster.isSelected()) { + if(storedMode == UserPreferences.SelectedMode.AUTOINGEST) { + needsRestart = true; + } + UserPreferences.setMode(UserPreferences.SelectedMode.STANDALONE); - //before return popup the message - if (needsRestart) { - SwingUtilities.invokeLater(() -> { - JOptionPane.showMessageDialog(null, - NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.MustRestart"), - NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.restartRequiredLabel.text"), - JOptionPane.WARNING_MESSAGE); - }); - } - return; } - - if (jRadioButtonAutomated.isSelected()) { - if (storedMode != UserPreferences.SelectedMode.AUTOINGEST) { - needsRestart = true; - } - String thePath = AutoIngestUserPreferences.getAutoModeImageFolder(); - if (thePath != null && 0 != inputPathTextField.getText().compareTo(thePath)) { - needsRestart = true; - } - thePath = AutoIngestUserPreferences.getAutoModeResultsFolder(); - if (thePath != null && 0 != outputPathTextField.getText().compareTo(thePath)) { + else if (jRadioButtonAutomated.isSelected()) { + if (storedMode == UserPreferences.SelectedMode.REVIEW) { needsRestart = true; } @@ -254,11 +235,7 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { AutoIngestUserPreferences.setSharedConfigMaster(masterNodeCheckBox.isSelected()); } } else if (jRadioButtonReview.isSelected()) { - if (storedMode != UserPreferences.SelectedMode.REVIEW) { - needsRestart = true; - } - String thePath = AutoIngestUserPreferences.getAutoModeResultsFolder(); - if (thePath != null && 0 != outputPathTextField.getText().compareTo(thePath)) { + if (storedMode == UserPreferences.SelectedMode.AUTOINGEST) { needsRestart = true; } @@ -661,7 +638,6 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { nodePanel = new javax.swing.JPanel(); jPanelNodeType = new javax.swing.JPanel(); jLabelSelectMode = new javax.swing.JLabel(); - restartRequiredNodeLabel = new javax.swing.JLabel(); jRadioButtonAutomated = new javax.swing.JRadioButton(); jRadioButtonReview = new javax.swing.JRadioButton(); jLabelSelectInputFolder = new javax.swing.JLabel(); @@ -703,9 +679,6 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { org.openide.awt.Mnemonics.setLocalizedText(jLabelSelectMode, org.openide.util.NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.jLabelSelectMode.text")); // NOI18N - restartRequiredNodeLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/experimental/images/warning16.png"))); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(restartRequiredNodeLabel, org.openide.util.NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.restartRequiredNodeLabel.text")); // NOI18N - modeRadioButtons.add(jRadioButtonAutomated); jRadioButtonAutomated.setSelected(true); org.openide.awt.Mnemonics.setLocalizedText(jRadioButtonAutomated, org.openide.util.NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.jRadioButtonAutomated.text")); // NOI18N @@ -777,10 +750,7 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { .addComponent(browseOutputFolderButton, javax.swing.GroupLayout.Alignment.TRAILING))) .addGroup(jPanelNodeTypeLayout.createSequentialGroup() .addGroup(jPanelNodeTypeLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(jPanelNodeTypeLayout.createSequentialGroup() - .addComponent(jLabelSelectMode) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(restartRequiredNodeLabel)) + .addComponent(jLabelSelectMode) .addComponent(jRadioButtonReview) .addComponent(jRadioButtonAutomated)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) @@ -802,9 +772,7 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { .addGroup(jPanelNodeTypeLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(jPanelNodeTypeLayout.createSequentialGroup() .addContainerGap() - .addGroup(jPanelNodeTypeLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(jLabelSelectMode) - .addComponent(restartRequiredNodeLabel)) + .addComponent(jLabelSelectMode) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(jRadioButtonAutomated) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -1407,7 +1375,6 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { jRadioButtonAutomated.setEnabled(enabled); jRadioButtonReview.setEnabled(enabled); outputPathTextField.setEnabled(enabled); - restartRequiredNodeLabel.setEnabled(enabled); } // Variables declaration - do not modify//GEN-BEGIN:variables @@ -1442,7 +1409,6 @@ public class AutoIngestSettingsPanel extends javax.swing.JPanel { private javax.swing.JScrollPane nodeScrollPane; private javax.swing.JTextField outputPathTextField; private javax.swing.JProgressBar pbTaskInProgress; - private javax.swing.JLabel restartRequiredNodeLabel; private javax.swing.JCheckBox sharedConfigCheckbox; private javax.swing.JTextField sharedSettingsErrorTextField; private javax.swing.JTextField sharedSettingsTextField; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java index f98eb1a909..8cd99b89c4 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/AutoIngestUserPreferences.java @@ -59,7 +59,7 @@ public final class AutoIngestUserPreferences { } /** - * Get "Join Automated Ingest Cluster" setting from persistent storage. + * Get "Join auto ingest cluster" setting from persistent storage. * * @return SelectedMode Selected setting. */ @@ -71,7 +71,7 @@ public final class AutoIngestUserPreferences { } /** - * Set "Join Automated Ingest Cluster" setting to persistent storage. + * Set "Join auto ingest cluster" setting to persistent storage. * * @param join boolean value of whether to join auto ingest cluster or not */ diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties index 21da10e84d..66ed50dd20 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/Bundle.properties @@ -25,7 +25,7 @@ AutoIngestSettingsPanel.AdvancedAutoIngestSettingsPanel.Title=Advanced Settings AutoIngestSettingsPanel.browseGlobalSettingsButton.text=Browse AutoIngestSettingsPanel.browseSharedSettingsButton.text=Browse AutoIngestSettingsPanel.CannotAccess=Cannot access -AutoIngestSettingsPanel.cbJoinAutoIngestCluster.text=Join Automated Ingest Cluster +AutoIngestSettingsPanel.cbJoinAutoIngestCluster.text=Join auto ingest cluster AutoIngestSettingsPanel.CheckPermissions=Check permissions. AutoIngestSettingsPanel.configButtonErrorTextField.text=configButtonErrorTextField AutoIngestSettingsPanel.downloadButton.text=Download Config @@ -89,7 +89,6 @@ OptionsDialog.jLabel1.text=jLabel1 StartupWindow.AutoIngestMode=Automated Ingest Node StartupWindow.CaseImportMode=Single User Case Import StartupWindow.CopyAndImportMode=Utilities -StartupWindow.ReviewMode=Cases StartupWindow.title.text=Welcome AdvancedAutoIngestSettingsPanel.lbInputScanIntervalMinutes.text=minutes AdvancedAutoIngestSettingsPanel.lbTimeoutHours.text=hour(s) @@ -133,10 +132,9 @@ AutoIngestSettingsPanel.inputPathTextField.toolTipText=Input folder for automate AutoIngestSettingsPanel.inputPathTextField.text= AutoIngestSettingsPanel.jLabelSelectInputFolder.text=Select shared images folder: AutoIngestSettingsPanel.jRadioButtonReview.toolTipText=Review cases created in automated processing mode -AutoIngestSettingsPanel.jRadioButtonReview.text=Examiner Node +AutoIngestSettingsPanel.jRadioButtonReview.text=Examiner node AutoIngestSettingsPanel.jRadioButtonAutomated.toolTipText=Automatically detect new data sources and create cases. -AutoIngestSettingsPanel.jRadioButtonAutomated.text=Auto Ingest Node -AutoIngestSettingsPanel.restartRequiredNodeLabel.text=Application restart required +AutoIngestSettingsPanel.jRadioButtonAutomated.text=Auto ingest node (application restart required) AutoIngestSettingsPanel.jLabelSelectMode.text=Select mode: AutoIngestSettingsPanel.jPanelIngestSettings.border.title=Automated Ingest Settings AutoIngestSettingsPanel.bnLogging.text=Node Status Logging diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/StartupWindow.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/StartupWindow.java index f36fa45f12..35d44dc407 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/StartupWindow.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/StartupWindow.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.experimental.configuration; -import java.awt.Cursor; import java.awt.Dimension; import java.awt.Toolkit; import java.awt.event.ActionEvent; @@ -36,7 +35,6 @@ import org.sleuthkit.autopsy.casemodule.StartupWindowInterface; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestControlPanel; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestCasePanel; /** * The default implementation of the Autopsy startup window @@ -48,7 +46,6 @@ public final class StartupWindow extends JDialog implements StartupWindowInterfa private static Dimension DIMENSIONS = new Dimension(750, 400); private static CueBannerPanel welcomeWindow; private static final long serialVersionUID = 1L; - private AutoIngestCasePanel caseManagementPanel = null; private static final String LOCAL_HOST_NAME = NetworkUtils.getLocalHostName(); public StartupWindow() { @@ -78,12 +75,6 @@ public final class StartupWindow extends JDialog implements StartupWindowInterfa @Override public void open() { - - if (caseManagementPanel != null) { - caseManagementPanel.updateView(); - caseManagementPanel.setCursor(Cursor.getDefaultCursor()); - } - if (welcomeWindow != null) { welcomeWindow.refresh(); } @@ -104,38 +95,27 @@ public final class StartupWindow extends JDialog implements StartupWindowInterfa * user. */ private void addPanelForMode() { - UserPreferences.SelectedMode mode = UserPreferences.getMode(); - - switch (mode) { - case AUTOINGEST: - this.setTitle(NbBundle.getMessage(StartupWindow.class, "StartupWindow.AutoIngestMode") + " (" + LOCAL_HOST_NAME + ")"); - setIconImage(ImageUtilities.loadImage("org/sleuthkit/autopsy/experimental/images/frame.gif", false)); //NON-NLS - this.addWindowListener(new WindowAdapter() { - @Override - public void windowClosing(WindowEvent e) { - AutoIngestControlPanel.getInstance().shutdown(); - } - }); - setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE); - add(AutoIngestControlPanel.getInstance()); - break; - case REVIEW: - this.setTitle(NbBundle.getMessage(StartupWindow.class, "StartupWindow.ReviewMode") + " (" + LOCAL_HOST_NAME + ")"); - caseManagementPanel = new AutoIngestCasePanel(this); - setIconImage(ImageUtilities.loadImage("org/sleuthkit/autopsy/experimental/images/frame.gif", false)); //NON-NLS - add(caseManagementPanel); - break; - default: - welcomeWindow = new CueBannerPanel(); - // add the command to close the window to the button on the Volume Detail Panel - welcomeWindow.setCloseButtonActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - close(); - } - }); - add(welcomeWindow); - break; + if(UserPreferences.getMode() == UserPreferences.SelectedMode.AUTOINGEST) { + this.setTitle(NbBundle.getMessage(StartupWindow.class, "StartupWindow.AutoIngestMode") + " (" + LOCAL_HOST_NAME + ")"); + setIconImage(ImageUtilities.loadImage("org/sleuthkit/autopsy/experimental/images/frame.gif", false)); //NON-NLS + this.addWindowListener(new WindowAdapter() { + @Override + public void windowClosing(WindowEvent e) { + AutoIngestControlPanel.getInstance().shutdown(); + } + }); + setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE); + add(AutoIngestControlPanel.getInstance()); + } else { + welcomeWindow = new CueBannerPanel(); + // add the command to close the window to the button on the Volume Detail Panel + welcomeWindow.setCloseButtonActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + close(); + } + }); + add(welcomeWindow); } } } \ No newline at end of file From bc3962514e5a831ea767d9260bdc1af8f0e36db7 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 5 Oct 2017 12:33:58 -0400 Subject: [PATCH 06/30] Various minor improvements. --- .../autopsy/casemodule/CueBannerPanel.java | 48 +++++++++++++++---- .../autoingest/AutoIngestCasePanel.java | 2 + 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.java index 2e639300f9..2d7b17458e 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/CueBannerPanel.java @@ -56,6 +56,12 @@ public class CueBannerPanel extends javax.swing.JPanel { } } + public static void closeAutoIngestCasesWindow() { + if (null != autoIngestCasePanelWindow) { + autoIngestCasePanelWindow.setVisible(false); + } + } + public CueBannerPanel() { initComponents(); customizeComponents(); @@ -84,8 +90,13 @@ public class CueBannerPanel extends javax.swing.JPanel { public void refresh() { enableComponents(); } - + private void customizeComponents() { + initRecentCasesWindow(); + initAutoIngestCasesWindow(); + } + + private void initRecentCasesWindow() { recentCasesWindow = new JDialog( WindowManager.getDefault().getMainWindow(), NbBundle.getMessage(CueBannerPanel.class, "CueBannerPanel.title.text"), @@ -107,9 +118,33 @@ public class CueBannerPanel extends javax.swing.JPanel { recentCasesWindow.pack(); recentCasesWindow.setResizable(false); } + + private void initAutoIngestCasesWindow() { + autoIngestCasePanelWindow = new JDialog( + WindowManager.getDefault().getMainWindow(), + REVIEW_MODE_TITLE, + Dialog.ModalityType.APPLICATION_MODAL); + autoIngestCasePanelWindow.getRootPane().registerKeyboardAction( + e -> { + autoIngestCasePanelWindow.setVisible(false); + }, + KeyStroke.getKeyStroke(KeyEvent.VK_ESCAPE, 0), JComponent.WHEN_IN_FOCUSED_WINDOW); + OpenRecentCasePanel recentCasesPanel = OpenRecentCasePanel.getInstance(); + recentCasesPanel.setCloseButtonActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + autoIngestCasePanelWindow.setVisible(false); + } + }); + AutoIngestCasePanelInterface autoIngestCasePanel = Lookup.getDefault().lookup(AutoIngestCasePanelInterface.class); + autoIngestCasePanel.addWindowStateListener(autoIngestCasePanelWindow); + autoIngestCasePanelWindow.add((JPanel)autoIngestCasePanel); + autoIngestCasePanelWindow.pack(); + autoIngestCasePanelWindow.setResizable(false); + } private void enableComponents() { - boolean enableOpenRecentCaseButton = (RecentCases.getInstance().getTotalRecentCases() == 0); + boolean enableOpenRecentCaseButton = (RecentCases.getInstance().getTotalRecentCases() > 0); openRecentCaseButton.setEnabled(enableOpenRecentCaseButton); openRecentCaseLabel.setEnabled(enableOpenRecentCaseButton); @@ -282,15 +317,8 @@ public class CueBannerPanel extends javax.swing.JPanel { }//GEN-LAST:event_openRecentCaseButtonActionPerformed private void openAutoIngestCaseButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_openAutoIngestCaseButtonActionPerformed - autoIngestCasePanelWindow = new JDialog(); - autoIngestCasePanelWindow.setTitle(REVIEW_MODE_TITLE); - AutoIngestCasePanelInterface autoIngestCasePanel = Lookup.getDefault().lookup(AutoIngestCasePanelInterface.class); - autoIngestCasePanel.addWindowStateListener(autoIngestCasePanelWindow); - autoIngestCasePanelWindow.add((JPanel)autoIngestCasePanel); - autoIngestCasePanelWindow.pack(); - autoIngestCasePanelWindow.toFront(); + autoIngestCasePanelWindow.setLocationRelativeTo(WindowManager.getDefault().getMainWindow()); autoIngestCasePanelWindow.setVisible(true); - autoIngestCasePanelWindow.setResizable(false); }//GEN-LAST:event_openAutoIngestCaseButtonActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java index 0adde4f773..e2519f20bd 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java @@ -49,6 +49,7 @@ import org.sleuthkit.autopsy.casemodule.StartupWindowProvider; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.casemodule.AutoIngestCasePanelInterface; +import org.sleuthkit.autopsy.casemodule.CueBannerPanel; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.experimental.configuration.StartupWindow; @@ -337,6 +338,7 @@ public final class AutoIngestCasePanel extends JPanel implements AutoIngestCaseP AutoIngestCaseManager.getInstance().openCase(caseMetadataFilePath); stopCasesTableRefreshes(); StartupWindowProvider.getInstance().close(); + CueBannerPanel.closeAutoIngestCasesWindow(); return null; } From dc06840e31eec0e72dc99c531a9f45e2d2bc3c15 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 5 Oct 2017 12:41:23 -0400 Subject: [PATCH 07/30] Cleanup. --- .../autoingest/AutoIngestCaseManager.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java index e7c0d49f93..32204fc612 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java @@ -18,21 +18,12 @@ */ package org.sleuthkit.autopsy.experimental.autoingest; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; -import javax.swing.SwingUtilities; -import org.openide.filesystems.FileObject; -import org.openide.filesystems.FileUtil; -import org.openide.util.actions.CallableSystemAction; -import org.sleuthkit.autopsy.casemodule.AddImageAction; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CaseActionException; -import org.sleuthkit.autopsy.casemodule.CaseNewAction; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.experimental.configuration.AutoIngestUserPreferences; /** @@ -40,7 +31,6 @@ import org.sleuthkit.autopsy.experimental.configuration.AutoIngestUserPreference */ final class AutoIngestCaseManager { - private static final Logger LOGGER = Logger.getLogger(AutoIngestCaseManager.class.getName()); private static AutoIngestCaseManager instance; /** @@ -60,9 +50,6 @@ final class AutoIngestCaseManager { * auto ingest. */ private AutoIngestCaseManager() { - - FileObject root = FileUtil.getConfigRoot(); - FileObject openRecentCasesMenu = root.getFileObject("Menu/Case/OpenRecentCase"); } /* From 9712f0329d8941e7dad82507e96854c0570f75ce Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 6 Oct 2017 13:25:42 -0400 Subject: [PATCH 08/30] Replaced folder scan with coordination service call. --- .../autoingest/AutoIngestCaseManager.java | 113 +++++++++++++++++- .../autoingest/AutoIngestCasePanel.java | 3 +- 2 files changed, 109 insertions(+), 7 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java index 32204fc612..1ba6ca35ba 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java @@ -22,8 +22,15 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.openide.util.NbBundle; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CaseActionException; +import org.sleuthkit.autopsy.coordinationservice.CoordinationService; +import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.experimental.configuration.AutoIngestUserPreferences; /** @@ -31,14 +38,17 @@ import org.sleuthkit.autopsy.experimental.configuration.AutoIngestUserPreference */ final class AutoIngestCaseManager { + private static final Logger LOGGER = Logger.getLogger(AutoIngestCaseManager.class.getName()); private static AutoIngestCaseManager instance; + + private CoordinationService coordinationService; /** * Gets the auto ingest case manager. * * @return The auto ingest case manager singleton. */ - synchronized static AutoIngestCaseManager getInstance() { + synchronized static AutoIngestCaseManager getInstance() throws AutoIngestCaseManager.AutoIngestCaseManagerException { if (null == instance) { instance = new AutoIngestCaseManager(); } @@ -49,21 +59,81 @@ final class AutoIngestCaseManager { * Constructs an object that handles locating and opening cases created by * auto ingest. */ - private AutoIngestCaseManager() { + private AutoIngestCaseManager() throws AutoIngestCaseManagerException { + try { + init(); + } catch (AutoIngestCaseManager.AutoIngestCaseManagerException ex) { + Logger.getLogger(AutoIngestCaseManager.class.getName()).log(Level.SEVERE, null, ex); + throw ex; + } + } + + /** + * Initialize the coordination service. + * + * @throws org.sleuthkit.autopsy.experimental.autoingest.AutoIngestCaseManager.AutoIngestCaseManagerException + */ + private void init() throws AutoIngestCaseManager.AutoIngestCaseManagerException { + try { + coordinationService = CoordinationService.getInstance(); + } catch (CoordinationServiceException ex) { + throw new AutoIngestCaseManager.AutoIngestCaseManagerException(ex.getMessage(), ex); + } } - /* + /** * Gets a list of the cases in the top level case folder used by auto * ingest. */ + @Messages({ + "AutoIngestCaseManager.CasePathsError=An error occurred while trying to retrieve the list of case paths." + }) List getCases() { List cases = new ArrayList<>(); - List caseFolders = PathUtils.findCaseFolders(Paths.get(AutoIngestUserPreferences.getAutoModeResultsFolder())); - for (Path caseFolderPath : caseFolders) { - cases.add(new AutoIngestCase(caseFolderPath)); + List casePathList; + try { + casePathList = getCasePaths(); + for (Path casePath : casePathList) { + cases.add(new AutoIngestCase(casePath)); + } + } catch (AutoIngestCaseManagerException ex) { + String errorMessage = NbBundle.getMessage(AutoIngestCaseManager.class, "AutoIngestCaseManager.CasePathsError"); + LOGGER.log(Level.SEVERE, errorMessage, ex); + MessageNotifyUtil.Message.error(errorMessage); } return cases; } + + /** + * Retrieve all of the case nodes and filter for only those that represent + * case paths. + * + * @return List of case paths. + */ + private List getCasePaths() throws AutoIngestCaseManagerException { + try { + List nodeList = coordinationService.getNodeList(CoordinationService.CategoryNode.CASES); + List casePathList = new ArrayList(0); + for (String node : nodeList) { + if(node.indexOf('\\') >= 0 || node.indexOf('/') >= 0) { + /* + * This is not a case name lock (name specifies a path). + */ + String nodeUpperCase = node.toUpperCase(); + if(!nodeUpperCase.endsWith("_RESOURCES") && !nodeUpperCase.endsWith("AUTO_INGEST_LOG.TXT")) { + /* + * This is not a case resource lock. Collect the path. + */ + casePathList.add(Paths.get(node)); + } + } + } + return casePathList; + + } catch (CoordinationServiceException ex) { + throw new AutoIngestCaseManager.AutoIngestCaseManagerException("Failed to get node list from coordination service.", ex); + } + } /** * Opens an auto ingest case case. @@ -78,4 +148,35 @@ final class AutoIngestCaseManager { */ Case.openAsCurrentCase(caseMetadataFilePath.toString()); } + + /** + * Exception type thrown when there is an error completing an auto ingest + * monitor operation. + */ + static final class AutoIngestCaseManagerException extends Exception { + + private static final long serialVersionUID = 1L; + + /** + * Constructs an instance of the exception type thrown when there is an + * error completing an auto ingest case manager operation. + * + * @param message The exception message. + */ + private AutoIngestCaseManagerException(String message) { + super(message); + } + + /** + * Constructs an instance of the exception type thrown when there is an + * error completing an auto ingest case manager operation. + * + * @param message The exception message. + * @param cause A Throwable cause for the error. + */ + private AutoIngestCaseManagerException(String message, Throwable cause) { + super(message, cause); + } + + } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java index e2519f20bd..2bcd335916 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCasePanel.java @@ -267,7 +267,8 @@ public final class AutoIngestCasePanel extends JPanel implements AutoIngestCaseP private void refreshCasesTable() { try { currentlySelectedCase = getSelectedCase(); - List theModel = AutoIngestCaseManager.getInstance().getCases(); + AutoIngestCaseManager manager = AutoIngestCaseManager.getInstance(); + List theModel = manager.getCases(); EventQueue.invokeLater(new CaseTableRefreshTask(theModel)); } catch (Exception ex) { logger.log(Level.SEVERE, "Unexpected exception in refreshCasesTable", ex); //NON-NLS From 91eaf88c10e7b2ade09291d0b09627de0c5f3045 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Oct 2017 14:34:19 -0400 Subject: [PATCH 09/30] Catching and handling exceptions caused by bad ZK node data so that they don't stop input folder scans --- .../autoingest/AutoIngestJob.java | 128 +++++++++++------- .../autoingest/AutoIngestManager.java | 22 +-- .../autoingest/AutoIngestMonitor.java | 2 +- 3 files changed, 97 insertions(+), 55 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java index 5e9bfbd26c..dd4cb60377 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java @@ -89,30 +89,34 @@ public final class AutoIngestJob implements Comparable, Serializa * * @param manifest The manifest for an automated ingest job. */ - AutoIngestJob(Manifest manifest) { - /* - * Version 0 fields. - */ - this.manifest = manifest; - this.nodeName = ""; - this.caseDirectoryPath = ""; - this.priority = DEFAULT_PRIORITY; - this.stage = Stage.PENDING; - this.stageStartDate = manifest.getDateFileCreated(); - this.dataSourceProcessor = null; - this.ingestJob = null; - this.cancelled = false; - this.completed = false; - this.completedDate = new Date(0); - this.errorsOccurred = false; + AutoIngestJob(Manifest manifest) throws AutoIngestJobException { + try { + /* + * Version 0 fields. + */ + this.manifest = manifest; + this.nodeName = ""; + this.caseDirectoryPath = ""; + this.priority = DEFAULT_PRIORITY; + this.stage = Stage.PENDING; + this.stageStartDate = manifest.getDateFileCreated(); + this.dataSourceProcessor = null; + this.ingestJob = null; + this.cancelled = false; + this.completed = false; + this.completedDate = new Date(0); + this.errorsOccurred = false; - /* - * Version 1 fields. - */ - this.version = CURRENT_VERSION; - this.processingStatus = ProcessingStatus.PENDING; - this.numberOfCrashes = 0; - this.stageDetails = this.getProcessingStageDetails(); + /* + * Version 1 fields. + */ + this.version = CURRENT_VERSION; + this.processingStatus = ProcessingStatus.PENDING; + this.numberOfCrashes = 0; + this.stageDetails = this.getProcessingStageDetails(); + } catch (Exception ex) { + throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex); + } } /** @@ -122,30 +126,34 @@ public final class AutoIngestJob implements Comparable, Serializa * @param nodeData The coordination service node data for an automated * ingest job. */ - AutoIngestJob(AutoIngestJobNodeData nodeData) { - /* - * Version 0 fields. - */ - this.manifest = new Manifest(nodeData.getManifestFilePath(), nodeData.getManifestFileDate(), nodeData.getCaseName(), nodeData.getDeviceId(), nodeData.getDataSourcePath(), Collections.emptyMap()); - this.nodeName = nodeData.getProcessingHostName(); - this.caseDirectoryPath = nodeData.getCaseDirectoryPath().toString(); - this.priority = nodeData.getPriority(); - this.stage = nodeData.getProcessingStage(); - this.stageStartDate = nodeData.getProcessingStageStartDate(); - this.dataSourceProcessor = null; // Transient data not in node data. - this.ingestJob = null; // Transient data not in node data. - this.cancelled = false; // Transient data not in node data. - this.completed = false; // Transient data not in node data. - this.completedDate = nodeData.getCompletedDate(); - this.errorsOccurred = nodeData.getErrorsOccurred(); + AutoIngestJob(AutoIngestJobNodeData nodeData) throws AutoIngestJobException { + try { + /* + * Version 0 fields. + */ + this.manifest = new Manifest(nodeData.getManifestFilePath(), nodeData.getManifestFileDate(), nodeData.getCaseName(), nodeData.getDeviceId(), nodeData.getDataSourcePath(), Collections.emptyMap()); + this.nodeName = nodeData.getProcessingHostName(); + this.caseDirectoryPath = nodeData.getCaseDirectoryPath().toString(); + this.priority = nodeData.getPriority(); + this.stage = nodeData.getProcessingStage(); + this.stageStartDate = nodeData.getProcessingStageStartDate(); + this.dataSourceProcessor = null; // Transient data not in node data. + this.ingestJob = null; // Transient data not in node data. + this.cancelled = false; // Transient data not in node data. + this.completed = false; // Transient data not in node data. + this.completedDate = nodeData.getCompletedDate(); + this.errorsOccurred = nodeData.getErrorsOccurred(); - /* - * Version 1 fields. - */ - this.version = CURRENT_VERSION; - this.processingStatus = nodeData.getProcessingStatus(); - this.numberOfCrashes = nodeData.getNumberOfCrashes(); - this.stageDetails = this.getProcessingStageDetails(); + /* + * Version 1 fields. + */ + this.version = CURRENT_VERSION; + this.processingStatus = nodeData.getProcessingStatus(); + this.numberOfCrashes = nodeData.getNumberOfCrashes(); + this.stageDetails = this.getProcessingStageDetails(); + } catch (Exception ex) { + throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex); + } } /** @@ -622,5 +630,33 @@ public final class AutoIngestJob implements Comparable, Serializa } } + + /** + * Exception thrown when there is a problem creating auto ingest job. + */ + final static class AutoIngestJobException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * Constructs an exception to throw when there is a problem creating + * auto ingest job. + * + * @param message The exception message. + */ + private AutoIngestJobException(String message) { + super(message); + } + + /** + * Constructs an exception to throw when there is a problem creating + * auto ingest job. + * + * @param message The exception message. + * @param cause The cause of the exception, if it was an exception. + */ + private AutoIngestJobException(String message, Throwable cause) { + super(message, cause); + } + } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 18d33b5c69..ee709fe544 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -61,6 +61,7 @@ import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; +import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; @@ -93,6 +94,7 @@ import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration; import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; +import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.AutoIngestJobException; import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason; import org.sleuthkit.autopsy.ingest.IngestJobSettings; @@ -759,7 +761,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang AutoIngestJob deletedJob = new AutoIngestJob(nodeData); deletedJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.DELETED); this.updateCoordinationServiceNode(deletedJob); - } catch (AutoIngestJobNodeData.InvalidDataException ex) { + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); return CaseDeletionResult.PARTIALLY_DELETED; } catch (InterruptedException | CoordinationServiceException ex) { @@ -1088,11 +1090,15 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus"); break; } - } catch (AutoIngestJobNodeData.InvalidDataException ex) { - SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); } } else { - addNewPendingJob(manifest); + try { + addNewPendingJob(manifest); + } catch (AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex); + } } } catch (CoordinationServiceException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex); @@ -1122,7 +1128,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * blocked, i.e., if auto ingest is * shutting down. */ - private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException { + private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException { AutoIngestJob job; if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) { job = new AutoIngestJob(nodeData); @@ -1176,7 +1182,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * blocked, i.e., if auto ingest is * shutting down. */ - private void addNewPendingJob(Manifest manifest) throws InterruptedException { + private void addNewPendingJob(Manifest manifest) throws InterruptedException, AutoIngestJobException { /* * Create the coordination service node data for the job. Note that * getting the lock will create the node for the job (with no data) @@ -1218,7 +1224,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * blocked, i.e., if auto ingest is * shutting down. */ - private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException { + private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException { /* * Try to get an exclusive lock on the coordination service node for * the job. If the lock cannot be obtained, another host in the auto @@ -1314,7 +1320,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * @throws CoordinationServiceException * @throws InterruptedException */ - private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException { + private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException, AutoIngestJobException { Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); if (null != caseDirectoryPath) { AutoIngestJob job; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index cec605f06b..9a262bb6d3 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -265,7 +265,7 @@ public final class AutoIngestMonitor extends Observable implements PropertyChang } } catch (InterruptedException ex) { LOGGER.log(Level.SEVERE, String.format("Unexpected interrupt while retrieving coordination service node data for '%s'", node), ex); - } catch (AutoIngestJobNodeData.InvalidDataException ex) { + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJob.AutoIngestJobException ex) { LOGGER.log(Level.SEVERE, String.format("Unable to use node data for '%s'", node), ex); } } From ca14274ec879df61e572e3b48fdf9f27e4338add Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Oct 2017 14:51:28 -0400 Subject: [PATCH 10/30] Added a catch-all firewal to visitFile() so that one bad file doesn't stop the entire input folder scan --- .../autoingest/AutoIngestManager.java | 143 +++++++++--------- 1 file changed, 75 insertions(+), 68 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index ee709fe544..c5f43a8f1f 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -1017,96 +1017,103 @@ public final class AutoIngestManager extends Observable implements PropertyChang * @return TERMINATE if auto ingest is shutting down, CONTINUE if it has * not. * - * @throws IOException if an I/O error occurs, but this implementation - * does not throw. */ @Override - public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) { if (Thread.currentThread().isInterrupted()) { return TERMINATE; } - Manifest manifest = null; - for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) { - if (parser.fileIsManifest(filePath)) { - try { - manifest = parser.parse(filePath); - break; - } catch (ManifestFileParserException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to parse %s with parser %s", filePath, parser.getClass().getCanonicalName()), ex); + try { + Manifest manifest = null; + for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) { + if (parser.fileIsManifest(filePath)) { + try { + manifest = parser.parse(filePath); + break; + } catch (ManifestFileParserException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to parse %s with parser %s", filePath, parser.getClass().getCanonicalName()), ex); + } + } + if (Thread.currentThread().isInterrupted()) { + return TERMINATE; } } + if (Thread.currentThread().isInterrupted()) { return TERMINATE; } - } - if (Thread.currentThread().isInterrupted()) { - return TERMINATE; - } - - if (null != manifest) { - /* + if (null != manifest) { + /* * Update the mapping of case names to manifest paths that is * used for case deletion. - */ - String caseName = manifest.getCaseName(); - Path manifestPath = manifest.getFilePath(); - if (casesToManifests.containsKey(caseName)) { - Set manifestPaths = casesToManifests.get(caseName); - manifestPaths.add(manifestPath); - } else { - Set manifestPaths = new HashSet<>(); - manifestPaths.add(manifestPath); - casesToManifests.put(caseName, manifestPaths); - } + */ + String caseName = manifest.getCaseName(); + Path manifestPath = manifest.getFilePath(); + if (casesToManifests.containsKey(caseName)) { + Set manifestPaths = casesToManifests.get(caseName); + manifestPaths.add(manifestPath); + } else { + Set manifestPaths = new HashSet<>(); + manifestPaths.add(manifestPath); + casesToManifests.put(caseName, manifestPaths); + } - /* + /* * Add a job to the pending jobs queue, the completed jobs list, * or do crashed job recovery, as required. - */ - try { - byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()); - if (null != rawData && rawData.length > 0) { - try { - AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData); - AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus(); - switch (processingStatus) { - case PENDING: - addPendingJob(manifest, nodeData); - break; - case PROCESSING: - doRecoveryIfCrashed(manifest, nodeData); - break; - case COMPLETED: - addCompletedJob(manifest, nodeData); - break; - case DELETED: - /* + */ + try { + byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()); + if (null != rawData && rawData.length > 0) { + try { + AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData); + AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus(); + switch (processingStatus) { + case PENDING: + addPendingJob(manifest, nodeData); + break; + case PROCESSING: + doRecoveryIfCrashed(manifest, nodeData); + break; + case COMPLETED: + addCompletedJob(manifest, nodeData); + break; + case DELETED: + /* * Ignore jobs marked as "deleted." - */ - break; - default: - SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus"); - break; + */ + break; + default: + SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus"); + break; + } + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); + } + } else { + try { + addNewPendingJob(manifest); + } catch (AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex); } - } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); - } - } else { - try { - addNewPendingJob(manifest); - } catch (AutoIngestJobException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex); } + } catch (CoordinationServiceException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex); + return CONTINUE; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return TERMINATE; } - } catch (CoordinationServiceException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex); - return CONTINUE; - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - return TERMINATE; } + + } catch (Exception ex) { + // Catch all unhandled and unexpected exceptions. Otherwise one bad file + // can stop the entire input folder scanning. Given that the exception is unexpected, + // I'm hesitant to add logging which requires accessing or de-referencing data. + SYS_LOGGER.log(Level.SEVERE, "Unexpected exception in file visitor", ex); + return CONTINUE; } if (!Thread.currentThread().isInterrupted()) { From 4739b8da933b7e0a6c1781a153581b5a8701ed2f Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 11 Oct 2017 16:11:36 -0400 Subject: [PATCH 11/30] Set build.type back to DEVELOPMENT in project.properties --- nbproject/project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbproject/project.properties b/nbproject/project.properties index da174330bb..ac6c347444 100755 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -6,8 +6,8 @@ app.name=${branding.token} ### if left unset, version will default to today's date app.version=4.5.0 ### build.type must be one of: DEVELOPMENT, RELEASE -build.type=RELEASE -#build.type=DEVELOPMENT +#build.type=RELEASE +build.type=DEVELOPMENT project.org.netbeans.progress=org-netbeans-api-progress project.org.sleuthkit.autopsy.experimental=Experimental From 49850bfd98c4a001a07983c8056c0855b3274fa2 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 11 Oct 2017 17:33:54 -0400 Subject: [PATCH 12/30] Further factored out the functionality into a separate class --- .../IdentifyDataSourceProcessors.java | 46 +++++++++++++++++++ .../autoingest/AutoIngestManager.java | 34 ++++++-------- 2 files changed, 60 insertions(+), 20 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/IdentifyDataSourceProcessors.java diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/IdentifyDataSourceProcessors.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/IdentifyDataSourceProcessors.java new file mode 100755 index 0000000000..9cdef77a7f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/IdentifyDataSourceProcessors.java @@ -0,0 +1,46 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.datasourceprocessors; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.openide.util.Lookup; +import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; + +/** + * A utility class to find Data Source Processors + */ +public class IdentifyDataSourceProcessors { + + /** + * A utility method to find all Data Source Processors (DSP) that are able + * to process the input data source. Only the DSPs that implement + * AutoIngestDataSourceProcessor interface are used. + * + * @param dataSourcePath Full path to the data source + * @return Hash map of all DSPs that can process the data source along with + * their confidence score + * @throws + * org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException + */ + public static Map getDataSourceProcessor(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { + + // lookup all AutomatedIngestDataSourceProcessors + Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); + + Map validDataSourceProcessorsMap = new HashMap<>(); + for (AutoIngestDataSourceProcessor processor : processorCandidates) { + int confidence = processor.canProcess(dataSourcePath); + if (confidence > 0) { + validDataSourceProcessorsMap.put(processor, confidence); + } + } + + return validDataSourceProcessorsMap; + } +} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 18d33b5c69..3dd17332d2 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -93,6 +93,7 @@ import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration; import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; +import org.sleuthkit.autopsy.datasourceprocessors.IdentifyDataSourceProcessors; import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason; import org.sleuthkit.autopsy.ingest.IngestJobSettings; @@ -2204,7 +2205,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang return; } - DataSource dataSource = identifyDataSource(caseForJob); + DataSource dataSource = identifyDataSource(); if (null == dataSource) { currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now())); return; @@ -2257,7 +2258,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * interrupted while blocked, i.e., * if auto ingest is shutting down. */ - private DataSource identifyDataSource(Case caseForJob) throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private DataSource identifyDataSource() throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath); @@ -2276,7 +2277,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang String deviceId = manifest.getDeviceId(); return new DataSource(deviceId, dataSourcePath); } - + /** * Passes the data source for the current job through a data source * processor that adds it to the case database. @@ -2306,21 +2307,14 @@ public final class AutoIngestManager extends Observable implements PropertyChang try { caseForJob.notifyAddingDataSource(taskId); - // lookup all AutomatedIngestDataSourceProcessors - Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); - - Map validDataSourceProcessorsMap = new HashMap<>(); - for (AutoIngestDataSourceProcessor processor : processorCandidates) { - try { - int confidence = processor.canProcess(dataSource.getPath()); - if (confidence > 0) { - validDataSourceProcessorsMap.put(processor, confidence); - } - } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { - SYS_LOGGER.log(Level.SEVERE, "Exception while determining whether data source processor {0} can process {1}", new Object[]{processor.getDataSourceType(), dataSource.getPath()}); - // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. - throw ex; - } + Map validDataSourceProcessorsMap; + try { + // lookup all AutomatedIngestDataSourceProcessors and poll which ones are able to process the current data source + validDataSourceProcessorsMap = IdentifyDataSourceProcessors.getDataSourceProcessor(dataSource.getPath()); + } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { + SYS_LOGGER.log(Level.SEVERE, "Exception while determining best data source processor for {0}", dataSource.getPath()); + // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. + throw ex; } // did we find a data source processor that can process the data source @@ -2650,7 +2644,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang } } - + /** * A data source processor progress monitor does nothing. There is * currently no mechanism for showing or recording data source processor @@ -3032,7 +3026,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang } } - + static final class AutoIngestManagerException extends Exception { private static final long serialVersionUID = 1L; From 3ac0792922ec0647f3d2ccf47ad16b9bf8c26e7d Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 12 Oct 2017 16:19:27 -0400 Subject: [PATCH 13/30] Refactored AddDataSourceCallback and DataSource out of AIM --- .../autoingest/AddDataSourceCallback.java | 85 +++++++++++++ .../autoingest/AutoIngestManager.java | 120 +----------------- .../experimental/autoingest/DataSource.java | 55 ++++++++ .../IdentifyDataSourceProcessors.java | 7 +- 4 files changed, 146 insertions(+), 121 deletions(-) create mode 100755 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java create mode 100755 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java rename {Core/src/org/sleuthkit/autopsy/datasourceprocessors => Experimental/src/org/sleuthkit/autopsy/experimental/autoingest}/IdentifyDataSourceProcessors.java (83%) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java new file mode 100755 index 0000000000..f566dffb73 --- /dev/null +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java @@ -0,0 +1,85 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.experimental.autoingest; + +import java.util.List; +import java.util.UUID; +import javax.annotation.concurrent.Immutable; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; +import org.sleuthkit.datamodel.Content; + + +/** + * A "callback" that collects the results of running a data source processor on + * a data source and unblocks the job processing thread when the data source + * processor finishes running in its own thread. + */ +@Immutable +class AddDataSourceCallback extends DataSourceProcessorCallback { + + private final Case caseForJob; + private final DataSource dataSourceInfo; + private final UUID taskId; + private final Object lock; + + /** + * Constructs a "callback" that collects the results of running a data + * source processor on a data source and unblocks the job processing thread + * when the data source processor finishes running in its own thread. + * + * @param caseForJob The case for the current job. + * @param dataSourceInfo The data source + * @param taskId The task id to associate with ingest job events. + */ + AddDataSourceCallback(Case caseForJob, DataSource dataSourceInfo, UUID taskId, Object lock) { + this.caseForJob = caseForJob; + this.dataSourceInfo = dataSourceInfo; + this.taskId = taskId; + this.lock = lock; + } + + /** + * Called by the data source processor when it finishes running in its own + * thread. + * + * @param result The result code for the processing of the data source. + * @param errorMessages Any error messages generated during the processing + * of the data source. + * @param dataSourceContent The content produced by processing the data + * source. + */ + @Override + public void done(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSourceContent) { + if (!dataSourceContent.isEmpty()) { + caseForJob.notifyDataSourceAdded(dataSourceContent.get(0), taskId); + } else { + caseForJob.notifyFailedAddingDataSource(taskId); + } + dataSourceInfo.setDataSourceProcessorOutput(result, errorMessages, dataSourceContent); + dataSourceContent.addAll(dataSourceContent); + synchronized (lock) { + lock.notify(); + } + } + + /** + * Called by the data source processor when it finishes running in its own + * thread, if that thread is the AWT (Abstract Window Toolkit) event + * dispatch thread (EDT). + * + * @param result The result code for the processing of the data source. + * @param errorMessages Any error messages generated during the processing + * of the data source. + * @param dataSourceContent The content produced by processing the data + * source. + */ + @Override + public void doneEDT(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSources) { + done(result, errorMessages, dataSources); + } + +} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 3dd17332d2..90fbf678e2 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -93,7 +93,6 @@ import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration; import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; -import org.sleuthkit.autopsy.datasourceprocessors.IdentifyDataSourceProcessors; import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason; import org.sleuthkit.autopsy.ingest.IngestJobSettings; @@ -2277,7 +2276,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang String deviceId = manifest.getDeviceId(); return new DataSource(deviceId, dataSourcePath); } - + /** * Passes the data source for the current job through a data source * processor that adds it to the case database. @@ -2300,7 +2299,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.INFO, "Adding data source for {0} ", manifestPath); currentJob.setProcessingStage(AutoIngestJob.Stage.ADDING_DATA_SOURCE, Date.from(Instant.now())); UUID taskId = UUID.randomUUID(); - DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId); + DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId, ingestLock); DataSourceProcessorProgressMonitor progressMonitor = new DoNothingDSPProgressMonitor(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, manifest.getDataSourceFileName(), caseDirectoryPath); @@ -2571,79 +2570,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang jobLogger.logFileExportError(); } } - - /** - * A "callback" that collects the results of running a data source - * processor on a data source and unblocks the job processing thread - * when the data source processor finishes running in its own thread. - */ - @Immutable - class AddDataSourceCallback extends DataSourceProcessorCallback { - - private final Case caseForJob; - private final DataSource dataSourceInfo; - private final UUID taskId; - - /** - * Constructs a "callback" that collects the results of running a - * data source processor on a data source and unblocks the job - * processing thread when the data source processor finishes running - * in its own thread. - * - * @param caseForJob The case for the current job. - * @param dataSourceInfo The data source - * @param taskId The task id to associate with ingest job - * events. - */ - AddDataSourceCallback(Case caseForJob, DataSource dataSourceInfo, UUID taskId) { - this.caseForJob = caseForJob; - this.dataSourceInfo = dataSourceInfo; - this.taskId = taskId; - } - - /** - * Called by the data source processor when it finishes running in - * its own thread. - * - * @param result The result code for the processing of - * the data source. - * @param errorMessages Any error messages generated during the - * processing of the data source. - * @param dataSourceContent The content produced by processing the - * data source. - */ - @Override - public void done(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSourceContent) { - if (!dataSourceContent.isEmpty()) { - caseForJob.notifyDataSourceAdded(dataSourceContent.get(0), taskId); - } else { - caseForJob.notifyFailedAddingDataSource(taskId); - } - dataSourceInfo.setDataSourceProcessorOutput(result, errorMessages, dataSourceContent); - dataSourceContent.addAll(dataSourceContent); - synchronized (ingestLock) { - ingestLock.notify(); - } - } - - /** - * Called by the data source processor when it finishes running in - * its own thread, if that thread is the AWT (Abstract Window - * Toolkit) event dispatch thread (EDT). - * - * @param result The result code for the processing of - * the data source. - * @param errorMessages Any error messages generated during the - * processing of the data source. - * @param dataSourceContent The content produced by processing the - * data source. - */ - @Override - public void doneEDT(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List dataSources) { - done(result, errorMessages, dataSources); - } - - } /** * A data source processor progress monitor does nothing. There is @@ -2984,48 +2910,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang PARTIALLY_DELETED, FULLY_DELETED } - - @ThreadSafe - private static final class DataSource { - - private final String deviceId; - private final Path path; - private DataSourceProcessorResult resultCode; - private List errorMessages; - private List content; - - DataSource(String deviceId, Path path) { - this.deviceId = deviceId; - this.path = path; - } - - String getDeviceId() { - return deviceId; - } - - Path getPath() { - return this.path; - } - - synchronized void setDataSourceProcessorOutput(DataSourceProcessorResult result, List errorMessages, List content) { - this.resultCode = result; - this.errorMessages = new ArrayList<>(errorMessages); - this.content = new ArrayList<>(content); - } - - synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode() { - return resultCode; - } - - synchronized List getDataSourceProcessorErrorMessages() { - return new ArrayList<>(errorMessages); - } - - synchronized List getContent() { - return new ArrayList<>(content); - } - - } static final class AutoIngestManagerException extends Exception { diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java new file mode 100755 index 0000000000..b655bfe0ae --- /dev/null +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java @@ -0,0 +1,55 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.experimental.autoingest; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.concurrent.ThreadSafe; +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult; +import org.sleuthkit.datamodel.Content; + +@ThreadSafe +class DataSource { + + private final String deviceId; + private final Path path; + private DataSourceProcessorResult resultCode; + private List errorMessages; + private List content; + + DataSource(String deviceId, Path path) { + this.deviceId = deviceId; + this.path = path; + } + + String getDeviceId() { + return deviceId; + } + + Path getPath() { + return this.path; + } + + synchronized void setDataSourceProcessorOutput(DataSourceProcessorResult result, List errorMessages, List content) { + this.resultCode = result; + this.errorMessages = new ArrayList<>(errorMessages); + this.content = new ArrayList<>(content); + } + + synchronized DataSourceProcessorResult getResultDataSourceProcessorResultCode() { + return resultCode; + } + + synchronized List getDataSourceProcessorErrorMessages() { + return new ArrayList<>(errorMessages); + } + + synchronized List getContent() { + return new ArrayList<>(content); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/IdentifyDataSourceProcessors.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java similarity index 83% rename from Core/src/org/sleuthkit/autopsy/datasourceprocessors/IdentifyDataSourceProcessors.java rename to Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java index 9cdef77a7f..d6673ad580 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/IdentifyDataSourceProcessors.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java @@ -3,19 +3,20 @@ * To change this template file, choose Tools | Templates * and open the template in the editor. */ -package org.sleuthkit.autopsy.datasourceprocessors; +package org.sleuthkit.autopsy.experimental.autoingest; import java.nio.file.Path; import java.util.Collection; import java.util.HashMap; import java.util.Map; import org.openide.util.Lookup; +import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; /** * A utility class to find Data Source Processors */ -public class IdentifyDataSourceProcessors { +class IdentifyDataSourceProcessors { /** * A utility method to find all Data Source Processors (DSP) that are able @@ -28,7 +29,7 @@ public class IdentifyDataSourceProcessors { * @throws * org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException */ - public static Map getDataSourceProcessor(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { + static Map getDataSourceProcessor(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { // lookup all AutomatedIngestDataSourceProcessors Collection processorCandidates = Lookup.getDefault().lookupAll(AutoIngestDataSourceProcessor.class); From 870b202b0f348ab406b649ec05ffea76e43e3009 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 12 Oct 2017 18:47:57 -0400 Subject: [PATCH 14/30] Delete obsolete externalresults package --- ...ampleExecutableDataSourceIngestModule.java | 394 ------------------ .../autopsy/externalresults/Bundle.properties | 27 -- .../externalresults/Bundle_ja.properties | 27 -- .../externalresults/ExternalResults.java | 212 ---------- .../ExternalResultsImporter.java | 308 -------------- .../ExternalResultsParser.java | 50 --- .../ExternalResultsXMLParser.java | 352 ---------------- .../autopsy_external_results.xsd | 70 ---- 8 files changed, 1440 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java delete mode 100755 Core/src/org/sleuthkit/autopsy/externalresults/Bundle.properties delete mode 100755 Core/src/org/sleuthkit/autopsy/externalresults/Bundle_ja.properties delete mode 100755 Core/src/org/sleuthkit/autopsy/externalresults/ExternalResults.java delete mode 100755 Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsImporter.java delete mode 100755 Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsParser.java delete mode 100755 Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsXMLParser.java delete mode 100755 Core/src/org/sleuthkit/autopsy/externalresults/autopsy_external_results.xsd diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java deleted file mode 100755 index f4f8060dd7..0000000000 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Sample module in the public domain. Feel free to use this as a template - * for your modules. - * - * Contact: Brian Carrier [carrier sleuthkit [dot] org] - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -package org.sleuthkit.autopsy.examples; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.coreutils.ErrorInfo; -import org.sleuthkit.autopsy.coreutils.ExecUtil; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.externalresults.ExternalResults; -import org.sleuthkit.autopsy.externalresults.ExternalResultsImporter; -import org.sleuthkit.autopsy.externalresults.ExternalResultsXMLParser; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModule; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; -import org.sleuthkit.autopsy.ingest.IngestJobContext; -import org.sleuthkit.autopsy.ingest.IngestMessage; -import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; -import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; -import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; -import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.Image; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -/** - * Sample data source ingest module that doesn't do much. Demonstrates use of - * utility classes: ExecUtils and the org.sleuthkit.autopsy.externalresults - * package. - */ -public class SampleExecutableDataSourceIngestModule implements DataSourceIngestModule { - - private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); - private static final String moduleName = SampleExecutableIngestModuleFactory.getModuleName(); - private final String fileInCaseDatabase = "/WINDOWS/system32/ntmsapi.dll"; // Probably - private IngestJobContext context; - private String outputDirPath; - private String derivedFileInCaseDatabase; - - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - this.context = context; - if (refCounter.incrementAndGet(context.getJobId()) == 1) { - // Create an output directory for this job. - outputDirPath = Case.getCurrentCase().getModuleDirectory() + File.separator + moduleName; //NON-NLS - File outputDir = new File(outputDirPath); - if (outputDir.exists() == false) { - outputDir.mkdirs(); - } - } - } - - @Override - public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar) { - if (refCounter.get(context.getJobId()) == 1) { - try { - // There will be two tasks: data source analysis and import of - // the results of the analysis. - progressBar.switchToDeterminate(2); - - // Do the analysis. The following sample code could be used to - // run an executable. In this case the executable would take - // two command line arguments, the path to the data source to be - // analyzed and the path to a results file to be generated. The - // results file would be an an XML file (see org.sleuthkit.autopsy.externalresults.autopsy_external_results.xsd) - // with instructions for the import of blackboard artifacts, - // derived files, and reports generated by the analysis. In this - // sample ingest module, the generation of the analysis results is - // simulated. - String resultsFilePath = outputDirPath + File.separator + String.format("job_%d_results.xml", context.getJobId()); - boolean haveRealExecutable = false; - if (haveRealExecutable) { - if (dataSource instanceof Image) { - Image image = (Image) dataSource; - String dataSourcePath = image.getPaths()[0]; - List commandLine = new ArrayList<>(); - commandLine.add("some.exe"); - commandLine.add(dataSourcePath); - commandLine.add(resultsFilePath); - ProcessBuilder processBuilder = new ProcessBuilder(commandLine); - ExecUtil.execute(processBuilder, new DataSourceIngestModuleProcessTerminator(context)); - } // not a disk image - else { - return ProcessResult.OK; - } - } else { - generateSimulatedResults(resultsFilePath); - } - progressBar.progress(1); - - // Import the results of the analysis. - ExternalResultsXMLParser resultsParser = new ExternalResultsXMLParser(dataSource, resultsFilePath); - ExternalResults results = resultsParser.parse(); - List errors = resultsParser.getErrorInfo(); - ExternalResultsImporter importer = new ExternalResultsImporter(); - errors.addAll(importer.importResults(results)); - for (ErrorInfo errorInfo : errors) { - IngestServices.getInstance().postMessage(IngestMessage.createErrorMessage(moduleName, "External Results Import Error", errorInfo.getMessage())); - } - progressBar.progress(2); - } catch (ParserConfigurationException | TransformerException | IOException ex) { - Logger logger = IngestServices.getInstance().getLogger(moduleName); - logger.log(Level.SEVERE, "Failed to simulate analysis and results import", ex); //NON-NLS - return ProcessResult.ERROR; - } - } - return ProcessResult.OK; - } - - private void generateSimulatedResults(String resultsFilePath) throws ParserConfigurationException, IOException, TransformerConfigurationException, TransformerException { - List derivedFilePaths = generateSimulatedDerivedFiles(); - List reportFilePaths = generateSimulatedReports(); - generateSimulatedResultsFile(derivedFilePaths, reportFilePaths, resultsFilePath); - } - - private List generateSimulatedDerivedFiles() throws IOException { - List filePaths = new ArrayList<>(); - String fileContents = "This is a simulated derived file."; - for (int i = 0; i < 2; ++i) { - String fileName = String.format("job_%d_derived_file_%d.txt", context.getJobId(), i); - filePaths.add(generateFile(fileName, fileContents.getBytes())); - if (i == 0) { - this.derivedFileInCaseDatabase = this.fileInCaseDatabase + "/" + fileName; - } - } - return filePaths; - } - - private List generateSimulatedReports() throws IOException { - List filePaths = new ArrayList<>(); - String fileContents = "This is a simulated report."; - for (int i = 0; i < 2; ++i) { - String fileName = String.format("job_%d_report_%d.txt", context.getJobId(), i); - filePaths.add(generateFile(fileName, fileContents.getBytes())); - } - return filePaths; - } - - private String generateFile(String fileName, byte[] fileContents) throws IOException { - String filePath = outputDirPath + File.separator + fileName; - File file = new File(filePath); - if (!file.exists()) { - file.createNewFile(); - } - try (FileOutputStream fileStream = new FileOutputStream(file)) { - fileStream.write(fileContents); - fileStream.flush(); - } - return filePath; - } - - private void generateSimulatedResultsFile(List derivedFilePaths, List reportPaths, String resultsFilePath) throws ParserConfigurationException, TransformerConfigurationException, TransformerException { - // SAMPLE GENERATED BY THE CODE BELOW: - // - // - // - // - // - // C:\cases\Small\ModuleOutput\Sample Executable Ingest Module\job_1_derived_file_0.txt - // /WINDOWS/system32/ntmsapi.dll - // - // - // C:\cases\Small\ModuleOutput\Sample Executable Ingest Module\job_1_derived_file_1.txt - // /WINDOWS/system32/ntmsapi.dll/job_1_derived_file_0.txt - // - // - // - // - // /WINDOWS/system32/ntmsapi.dll - // - // SampleInterestingFilesSet - // Sample Executable Ingest Module - // - // - // - // /WINDOWS/system32/ntmsapi.dll/job_1_derived_file_0.txt - // - // One - // - // - // 2 - // - // - // 3 - // - // - // 4.0 - // - // - // - // - // - // C:\cases\Small\ModuleOutput\Sample Executable Ingest Module\job_1_report_0.txt - // Sample Executable Ingest Module - // Sample Report - // - // - // C:\cases\Small\ModuleOutput\Sample Executable Ingest Module\job_1_report_1.txt - // Sample Executable Ingest Module - // - // - // - - // Create the XML DOM document and the root element. - DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder(); - Document doc = docBuilder.newDocument(); - Element rootElement = doc.createElement(ExternalResultsXMLParser.TagNames.ROOT_ELEM.toString()); - doc.appendChild(rootElement); - - // Add a derived files list element to the root element. - Element derivedFilesListElement = doc.createElement(ExternalResultsXMLParser.TagNames.DERIVED_FILES_LIST_ELEM.toString()); - rootElement.appendChild(derivedFilesListElement); - - // Add derived file elements to the derived files list element. Each - // file element gets required local path and parent file child elements. - // Note that the local path of the derived file must be to a location in - // the case directory or a subdirectory of the case directory and the - // parent file must be specified using the path format used in the case - // database, e.g., /WINDOWS/system32/ntmsapi.dll, where volume, file - // system, etc. are not in the path. - for (int i = 0; i < derivedFilePaths.size(); ++i) { - String filePath = derivedFilePaths.get(i); - Element derivedFileElement = doc.createElement(ExternalResultsXMLParser.TagNames.DERIVED_FILE_ELEM.toString()); - derivedFilesListElement.appendChild(derivedFileElement); - Element localPathElement = doc.createElement(ExternalResultsXMLParser.TagNames.LOCAL_PATH_ELEM.toString()); - localPathElement.setTextContent(filePath); - derivedFileElement.appendChild(localPathElement); - Element parentPathElement = doc.createElement(ExternalResultsXMLParser.TagNames.PARENT_FILE_ELEM.toString()); - if (i == 0) { - parentPathElement.setTextContent(this.fileInCaseDatabase); - } else { - parentPathElement.setTextContent(this.derivedFileInCaseDatabase); - } - derivedFileElement.appendChild(parentPathElement); - } - - // Add an artifacts list element to the root element. - Element artifactsListElement = doc.createElement(ExternalResultsXMLParser.TagNames.ARTIFACTS_LIST_ELEM.toString()); - rootElement.appendChild(artifactsListElement); - - // Add an artifact element to the artifacts list element with the required - // artifact type attribute. A standard artifact type is used as the type - // attribute of this artifact element. - Element artifactElement = doc.createElement(ExternalResultsXMLParser.TagNames.ARTIFACT_ELEM.toString()); - artifactElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getLabel()); - artifactsListElement.appendChild(artifactElement); - - // Add the required source file element to the artifact element. Note - // that source file must be either the local path of a derived file or a - // file in the case database. - Element fileElement = doc.createElement(ExternalResultsXMLParser.TagNames.SOURCE_FILE_ELEM.toString()); - fileElement.setTextContent(this.fileInCaseDatabase); - artifactElement.appendChild(fileElement); - - // Add an artifact attribute element to the artifact element. A standard - // artifact attribute type is used as the required type XML attribute of - // the artifact attribute element. - Element artifactAttrElement = doc.createElement(ExternalResultsXMLParser.TagNames.ATTRIBUTE_ELEM.toString()); - artifactAttrElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), ATTRIBUTE_TYPE.TSK_SET_NAME.getLabel()); - artifactElement.appendChild(artifactAttrElement); - - // Add the required value element to the artifact attribute element, - // with an optional type XML attribute of ExternalXML.VALUE_TYPE_TEXT, - // which is the default. - Element artifactAttributeValueElement = doc.createElement(ExternalResultsXMLParser.TagNames.VALUE_ELEM.toString()); - artifactAttributeValueElement.setTextContent("SampleInterestingFilesSet"); - artifactAttrElement.appendChild(artifactAttributeValueElement); - - // Add an optional source module element to the artifact attribute - // element. - Element artifactAttrSourceElement = doc.createElement(ExternalResultsXMLParser.TagNames.SOURCE_MODULE_ELEM.toString()); - artifactAttrSourceElement.setTextContent(moduleName); - artifactAttrElement.appendChild(artifactAttrSourceElement); - - // Add an artifact element with a user-defined type. - artifactElement = doc.createElement(ExternalResultsXMLParser.TagNames.ARTIFACT_ELEM.toString()); - artifactElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), "SampleArtifactType"); - artifactsListElement.appendChild(artifactElement); - - // Add the required source file element. - fileElement = doc.createElement(ExternalResultsXMLParser.TagNames.SOURCE_FILE_ELEM.toString()); - fileElement.setTextContent(this.derivedFileInCaseDatabase); - artifactElement.appendChild(fileElement); - - // Add artifact attribute elements with user-defined types to the - // artifact element, adding value elements of assorted types. - for (int i = 0; i < 5; ++i) { - artifactAttrElement = doc.createElement(ExternalResultsXMLParser.TagNames.ATTRIBUTE_ELEM.toString()); - artifactAttrElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), "SampleArtifactAttributeType" + i); - artifactElement.appendChild(artifactAttrElement); - artifactAttributeValueElement = doc.createElement(ExternalResultsXMLParser.TagNames.VALUE_ELEM.toString()); - switch (i) { - case 0: - artifactAttributeValueElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), ExternalResultsXMLParser.AttributeValues.VALUE_TYPE_TEXT.toString()); - artifactAttributeValueElement.setTextContent("One"); - break; - case 1: - artifactAttributeValueElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), ExternalResultsXMLParser.AttributeValues.VALUE_TYPE_INT32.toString()); - artifactAttributeValueElement.setTextContent("2"); - break; - case 2: - artifactAttributeValueElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), ExternalResultsXMLParser.AttributeValues.VALUE_TYPE_INT64.toString()); - artifactAttributeValueElement.setTextContent("3"); - break; - case 3: - artifactAttributeValueElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), ExternalResultsXMLParser.AttributeValues.VALUE_TYPE_DOUBLE.toString()); - artifactAttributeValueElement.setTextContent("4.0"); - break; - case 4: - artifactAttributeValueElement.setAttribute(ExternalResultsXMLParser.AttributeNames.TYPE_ATTR.toString(), ExternalResultsXMLParser.AttributeValues.VALUE_TYPE_DATETIME.toString()); - artifactAttributeValueElement.setTextContent("7023372036854775839"); - break; - } - artifactAttrElement.appendChild(artifactAttributeValueElement); - } - - // Add a reports list element to the root element. - Element reportsListElement = doc.createElement(ExternalResultsXMLParser.TagNames.REPORTS_LIST_ELEM.toString()); - rootElement.appendChild(reportsListElement); - - // Add report elements to the reports list element. Each report element - // gets required local path and source module child elements. There is - // also an optional report name element. Note that the local path of the - // report must be to a location in the case directory or a subdirectory - // of the case directory and the parent file must be specified using the - // path format used in the case database, e.g., /WINDOWS/system32/ntmsapi.dll, - // where volume, file system, etc. are not in the path. - for (int i = 0; i < reportPaths.size(); ++i) { - String reportPath = reportPaths.get(i); - Element reportElement = doc.createElement(ExternalResultsXMLParser.TagNames.REPORT_ELEM.toString()); - reportsListElement.appendChild(reportElement); - Element reportPathElement = doc.createElement(ExternalResultsXMLParser.TagNames.LOCAL_PATH_ELEM.toString()); - reportPathElement.setTextContent(reportPath); - reportElement.appendChild(reportPathElement); - Element reportSourceModuleElement = doc.createElement(ExternalResultsXMLParser.TagNames.SOURCE_MODULE_ELEM.toString()); - reportSourceModuleElement.setTextContent(moduleName); - reportElement.appendChild(reportSourceModuleElement); - if (i == 0) { - Element reportNameElement = doc.createElement(ExternalResultsXMLParser.TagNames.REPORT_NAME_ELEM.toString()); - reportNameElement.setTextContent("Sample Report"); - reportElement.appendChild(reportNameElement); - } - } - - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - DOMSource source = new DOMSource(doc); - StreamResult result = new StreamResult(new File(resultsFilePath)); - transformer.transform(source, result); - } -} diff --git a/Core/src/org/sleuthkit/autopsy/externalresults/Bundle.properties b/Core/src/org/sleuthkit/autopsy/externalresults/Bundle.properties deleted file mode 100755 index 94ded0f3d5..0000000000 --- a/Core/src/org/sleuthkit/autopsy/externalresults/Bundle.properties +++ /dev/null @@ -1,27 +0,0 @@ -ExternalResultsImporter.importDerivedFiles.errMsg1.text=Could not import derived file at {0}, parent file {1} not found. -ExternalResultsImporter.importDerivedFiles.errMsg2.text=Could not import derived file at {0}, file does not exist. -ExternalResultsImporter.importDerivedFiles.errMsg3.text=Could not import derived file at {0}, error querying/updating case database. -ExternalResultsImporter.importArtifacts.caseErrMsg1.text=Could not import {0} attribute, value \= {1}, for {2} artifact from {3}, unrecognized attribute value type\: {4}. -ExternalResultsImporter.importArtifacts.errMsg1.text=Could not import {0} artifact from {1}, source file not found. -ExternalResultsImporter.importArtifacts.errMsg2.text=Could not import {0} artifact from {1}, error updating case database. -ExternalResultsImporter.importReports.errMsg1.text=Could not import report at {0}, file does not exist. -ExternalResultsImporter.importReports.errMsg2.text=Could not import report at {0}, error updating case database. -ExternalResultsImporter.findFileInCaseDatabase.errMsg1.text=Parent file path {0} is ambiguous, using first file found. -ExternalResultsImporter.getPathRelativeToCaseFolder.errMsg1.text=Did not convert {0} to relative path, not in a subdirectory of case directory {1}. -ExternalResultsImporter.getPathRelativeToCaseFolder.errMsg2.text=Expected {0} to be an absolute path to a file in a subdirectory of case directory {1}. -ExternalResultsXMLParser.parse.errMsg1.text=Did not find {0} root element of {1}. -ExternalResultsXMLParser.parse.errMsg2.text=Error parsing {0}. -ExternalResultsXMLParser.parseArtifactAttributes.errMsg1.text=Found {0} element that has no content in {1}. -ExternalResultsXMLParser.parseAttributeValueType.errMsg1.text=Found unrecognized value {0} for {1} attribute of {2} element. -ExternalResultsXMLParser.getChildElementContent.errMsg1.text=Found {0} element with {1} child element that has no content in {2}. -ExternalResultsXMLParser.getChildElementContent.errMsg2.text=Found {0} element missing {1} child element in {2}. -ExternalResultsXMLParser.getChildElement.errMsg1.text=Found multiple {0} child elements for {1} element in {2}, ignoring all but first occurrence. -ExternalResults.addArtifact.exception.msg1.text=type argument is empty. -ExternalResults.addArtifact.exception.msg2.text=source argument is empty. -ExternalResults.addReport.exception.msg1.text=localPath argument is empty. -ExternalResults.addReport.exception.msg2.text=sourceModuleName argument is empty. -ExternalResults.addDerivedFile.exception.msg1.text=localPath argument is empty. -ExternalResults.addDerivedFile.exception.msg2.text=parentPath argument is empty. -ExternalResults.Artifact.addAttribute.exception.msg1.text=type argument is empty. -ExternalResults.Artifact.addAttribute.exception.msg2.text=value argument is empty. -ExternalResults.Artifact.addAttribute.exception.msg3.text=valueType argument is empty. \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/externalresults/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/externalresults/Bundle_ja.properties deleted file mode 100755 index b9702f4b10..0000000000 --- a/Core/src/org/sleuthkit/autopsy/externalresults/Bundle_ja.properties +++ /dev/null @@ -1,27 +0,0 @@ -ExternalResults.addArtifact.exception.msg1.text=\u5F15\u6570\u30BF\u30A4\u30D7\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResults.addArtifact.exception.msg2.text=\u30BD\u30FC\u30B9\u5F15\u6570\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResults.addDerivedFile.exception.msg1.text=\u30ED\u30FC\u30AB\u30EB\u30D1\u30B9\u5F15\u6570\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResults.addDerivedFile.exception.msg2.text=\u30DA\u30A2\u30EC\u30F3\u30C8\u30D1\u30B9\u5F15\u6570\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResults.addReport.exception.msg1.text=\u30ED\u30FC\u30AB\u30EB\u30D1\u30B9\u5F15\u6570\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResults.addReport.exception.msg2.text=\u30BD\u30FC\u30B9\u30E2\u30B8\u30E5\u30FC\u30EB\u540D\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResults.Artifact.addAttribute.exception.msg1.text=\u5F15\u6570\u30BF\u30A4\u30D7\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResults.Artifact.addAttribute.exception.msg2.text=\u5F15\u6570\u5024\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResultsImporter.findFileInCaseDatabase.errMsg1.text=\u30DA\u30A2\u30EC\u30F3\u30C8\u30D5\u30A1\u30A4\u30EB\u30D1\u30B9{0}\u304C\u66D6\u6627\u3067\u3059\u3002\u6700\u521D\u306B\u898B\u3064\u3051\u305F\u30D5\u30A1\u30A4\u30EB\u3092\u5229\u7528\u3057\u307E\u3059\u3002 -ExternalResultsImporter.getPathRelativeToCaseFolder.errMsg1.text={0}\u3092\u76F8\u5BFE\u30D1\u30B9\u306B\u5909\u63DB\u3057\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30B1\u30FC\u30B9\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA{1}\u306E\u30B5\u30D6\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u306B\u3042\u308A\u307E\u305B\u3093\u3002 -ExternalResultsImporter.getPathRelativeToCaseFolder.errMsg2.text={0}\u306F\u30B1\u30FC\u30B9\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA{1}\u306E\u30B5\u30D6\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u5185\u306E\u30D5\u30A1\u30A4\u30EB\u3078\u306E\u7D76\u5BFE\u30D1\u30B9\u3068\u4E88\u60F3\u3055\u308C\u3066\u3044\u307E\u3059\u3002 -ExternalResultsImporter.importArtifacts.caseErrMsg1.text={3}\u306E{2}\u30A2\u30FC\u30C6\u30A3\u30D5\u30A1\u30AF\u30C8\u306E{0}\u5C5E\u6027\u3001\u30D0\u30EA\u30E5\u30FC\uFF1D{1}\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u8A8D\u8B58\u3055\u308C\u3066\u3044\u306A\u3044\u5C5E\u6027\u5024\u578B\uFF1A{4} -ExternalResultsImporter.importDerivedFiles.errMsg1.text={0}\u306B\u6D3E\u751F\u30D5\u30A1\u30A4\u30EB\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30DA\u30A2\u30EC\u30F3\u30C8\u30D5\u30A1\u30A4\u30EB{1}\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F\u3002 -ExternalResultsImporter.importDerivedFiles.errMsg2.text={0}\u306B\u6D3E\u751F\u30D5\u30A1\u30A4\u30EB\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30D5\u30A1\u30A4\u30EB\u304C\u5B58\u5728\u3057\u307E\u305B\u3093\u3002 -ExternalResultsImporter.importDerivedFiles.errMsg3.text={0}\u306B\u6D3E\u751F\u30D5\u30A1\u30A4\u30EB\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30B1\u30FC\u30B9\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u3067\u306E\u30AF\u30A8\u30EA\u30FC\uFF0F\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u306E\u5B9F\u884C\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002 -ExternalResultsImporter.importReports.errMsg1.text={0}\u306B\u30EC\u30DD\u30FC\u30C8\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30D5\u30A1\u30A4\u30EB\u304C\u5B58\u5728\u3057\u307E\u305B\u3093\u3002 -ExternalResultsImporter.importReports.errMsg2.text={0}\u306B\u30EC\u30DD\u30FC\u30C8\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30B1\u30FC\u30B9\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002 -ExternalResultsXMLParser.parse.errMsg2.text={0}\u306E\u30D1\u30FC\u30B9\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002 -ExternalResults.Artifact.addAttribute.exception.msg3.text=\u5024\u578B\u5F15\u6570\u304C\u7A7A\u767D\u3067\u3059\u3002 -ExternalResultsImporter.importArtifacts.errMsg1.text={1}\u306E{0}\u30A2\u30FC\u30C6\u30A3\u30D5\u30A1\u30AF\u30C8\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30BD\u30FC\u30B9\u30D5\u30A1\u30A4\u30EB\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F\u3002 -ExternalResultsImporter.importArtifacts.errMsg2.text={1}\u306E{0}\u30A2\u30FC\u30C6\u30A3\u30D5\u30A1\u30AF\u30C8\u3092\u30A4\u30F3\u30DD\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30B1\u30FC\u30B9\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002 -ExternalResultsXMLParser.getChildElement.errMsg1.text={2}\u306B\u3066\u8907\u6570\u306E{1}\u30A8\u30EC\u30E1\u30F3\u30C8\u306E{0}\u30C1\u30E3\u30A4\u30EB\u30C9\u30A8\u30EC\u30E1\u30F3\u30C8\u304C\u898B\u3064\u304B\u308A\u307E\u3057\u305F\u3002\u4E00\u756A\u76EE\u4EE5\u5916\u306F\u7121\u8996\u3057\u307E\u3059\u3002 -ExternalResultsXMLParser.getChildElementContent.errMsg1.text={2}\u306B\u30B3\u30F3\u30C6\u30F3\u30C4\u304C\u306A\u304F\u3001{1}\u30C1\u30E3\u30A4\u30EB\u30C9\u30A8\u30EC\u30E1\u30F3\u30C8\u304C\u3042\u308B\u3001{0}\u30A8\u30EC\u30E1\u30F3\u30C8\u304C\u898B\u3064\u304B\u308A\u307E\u3057\u305F\u3002 -ExternalResultsXMLParser.getChildElementContent.errMsg2.text={2}\u306B\u3066{1}\u30C1\u30E3\u30A4\u30EB\u30C9\u30A8\u30EC\u30E1\u30F3\u30C8\u304C\u306A\u3044{0}\u30A8\u30EC\u30E1\u30F3\u30C8\u304C\u898B\u3064\u304B\u308A\u307E\u3057\u305F\u3002 -ExternalResultsXMLParser.parse.errMsg1.text={1}\u306E{0}\u30EB\u30FC\u30C8\u30A8\u30EC\u30E1\u30F3\u30C8\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093\u3067\u3057\u305F\u3002 -ExternalResultsXMLParser.parseArtifactAttributes.errMsg1.text={1}\u306B\u30B3\u30F3\u30C6\u30F3\u30C4\u304C\u306A\u3044{0}\u30A8\u30EC\u30E1\u30F3\u30C8\u304C\u898B\u3064\u304B\u308A\u307E\u3057\u305F\u3002 -ExternalResultsXMLParser.parseAttributeValueType.errMsg1.text={2}\u30A8\u30EC\u30E1\u30F3\u30C8\u306E{1}\u5C5E\u6027\u306B\u5BFE\u3057\u3066\u3001\u8A8D\u8B58\u3055\u308C\u306A\u3044\u30D0\u30EA\u30E5\u30FC{0}\u304C\u898B\u3064\u304B\u308A\u307E\u3057\u305F\u3002 \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResults.java b/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResults.java deleted file mode 100755 index 153d4acda8..0000000000 --- a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResults.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.externalresults; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.openide.util.NbBundle; -import org.sleuthkit.datamodel.Content; - -/** - * - */ -final public class ExternalResults { - - private final Content dataSource; - private final List artifacts = new ArrayList<>(); - private final List reports = new ArrayList<>(); - private final List derivedFiles = new ArrayList<>(); - - ExternalResults(Content dataSource) { - this.dataSource = dataSource; - } - - Content getDataSource() { - return this.dataSource; - } - - Artifact addArtifact(String type, String sourceFilePath) { - if (type.isEmpty()) { - throw new IllegalArgumentException( - NbBundle.getMessage(this.getClass(), "ExternalResults.addArtifact.exception.msg1.text")); - } - if (sourceFilePath.isEmpty()) { - throw new IllegalArgumentException( - NbBundle.getMessage(this.getClass(), "ExternalResults.addArtifact.exception.msg2.text")); - } - Artifact artifact = new Artifact(type, sourceFilePath); - artifacts.add(artifact); - return artifact; - } - - List getArtifacts() { - return Collections.unmodifiableList(artifacts); - } - - void addReport(String localPath, String sourceModuleName, String reportName) { - if (localPath.isEmpty()) { - throw new IllegalArgumentException( - NbBundle.getMessage(this.getClass(), "ExternalResults.addReport.exception.msg1.text")); - } - if (sourceModuleName.isEmpty()) { - throw new IllegalArgumentException( - NbBundle.getMessage(this.getClass(), "ExternalResults.addReport.exception.msg2.text")); - } - Report report = new Report(localPath, sourceModuleName, reportName); - reports.add(report); - } - - List getReports() { - return Collections.unmodifiableList(reports); - } - - void addDerivedFile(String localPath, String parentPath) { - if (localPath.isEmpty()) { - throw new IllegalArgumentException( - NbBundle.getMessage(this.getClass(), "ExternalResults.addDerivedFile.exception.msg1.text")); - } - if (parentPath.isEmpty()) { - throw new IllegalArgumentException( - NbBundle.getMessage(this.getClass(), "ExternalResults.addDerivedFile.exception.msg2.text")); - } - DerivedFile file = new DerivedFile(localPath, parentPath); - derivedFiles.add(file); - } - - List getDerivedFiles() { - return Collections.unmodifiableList(derivedFiles); - } - - static final class Artifact { - - private final String type; - private final String sourceFilePath; - private final ArrayList attributes = new ArrayList<>(); - - Artifact(String type, String sourceFilePath) { - this.type = type; - this.sourceFilePath = sourceFilePath; - } - - String getType() { - return type; - } - - String getSourceFilePath() { - return sourceFilePath; - } - - void addAttribute(String type, String value, String valueType, String sourceModule) { - if (type.isEmpty()) { - throw new IllegalArgumentException(NbBundle.getMessage(this.getClass(), - "ExternalResults.Artifact.addAttribute.exception.msg1.text")); - } - if (value.isEmpty()) { - throw new IllegalArgumentException(NbBundle.getMessage(this.getClass(), - "ExternalResults.Artifact.addAttribute.exception.msg2.text")); - } - if (valueType.isEmpty()) { - throw new IllegalArgumentException(NbBundle.getMessage(this.getClass(), - "ExternalResults.Artifact.addAttribute.exception.msg3.text")); - } - attributes.add(new ArtifactAttribute(type, value, valueType, sourceModule)); - } - - List getAttributes() { - return Collections.unmodifiableList(attributes); - } - } - - static final class ArtifactAttribute { - - private final String type; - private final String valueType; - private final String value; - private final String sourceModule; - - private ArtifactAttribute(String type, String value, String valueType, String sourceModule) { - this.type = type; - this.value = value; - this.valueType = valueType; - this.sourceModule = sourceModule; - } - - String getType() { - return type; - } - - String getValue() { - return value; - } - - String getValueType() { - return valueType; - } - - String getSourceModule() { - return sourceModule; - } - } - - static final class Report { - - private final String localPath; - private final String sourceModuleName; - private final String reportName; - - Report(String localPath, String sourceModuleName, String displayName) { - this.localPath = localPath; - this.sourceModuleName = sourceModuleName; - this.reportName = displayName; - } - - String getLocalPath() { - return localPath; - } - - String getSourceModuleName() { - return sourceModuleName; - } - - String getReportName() { - return reportName; - } - } - - static final class DerivedFile { - - private final String localPath; - private final String parentPath; - - DerivedFile(String localPath, String parentPath) { - this.localPath = localPath; - this.parentPath = parentPath; - } - - String getLocalPath() { - return localPath; - } - - String getParentPath() { - return parentPath; - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsImporter.java b/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsImporter.java deleted file mode 100755 index 9fc6b4927f..0000000000 --- a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsImporter.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011-2016 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this localFile except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.externalresults; - -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.logging.Level; -import org.openide.util.NbBundle; -import org.openide.util.NbBundle.Messages; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.services.Blackboard; -import org.sleuthkit.autopsy.casemodule.services.FileManager; -import org.sleuthkit.autopsy.coreutils.ErrorInfo; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.autopsy.ingest.ModuleContentEvent; -import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DerivedFile; -import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.TskDataException; - -/** - * Uses a standard representation of results data (e.g., artifacts, derived - * files, reports) to import results generated by a process external to Autopsy - * into Autopsy. - */ -public final class ExternalResultsImporter { - - private static final Logger logger = Logger.getLogger(ExternalResultsImporter.class.getName()); - private static final HashSet standardArtifactTypeIds = new HashSet<>(); - private final List errors = new ArrayList<>(); - private Blackboard blackboard; - - static { - for (BlackboardArtifact.ARTIFACT_TYPE artifactType : BlackboardArtifact.ARTIFACT_TYPE.values()) { - standardArtifactTypeIds.add(artifactType.getTypeID()); - } - } - - /** - * Import results generated by a process external to Autopsy into Autopsy. - * - * @param results A standard representation of results data (e.g., - * artifacts, derived files, reports)from the data source. - * - * @return A collection of error messages, possibly empty. The error - * messages are already logged but are provided to allow the caller - * to provide additional user feedback via the Autopsy user - * interface. - */ - public List importResults(ExternalResults results) { - blackboard = Case.getCurrentCase().getServices().getBlackboard(); - // Import files first, they may be artifactData sources. - importDerivedFiles(results); - importArtifacts(results); - importReports(results); - List importErrors = new ArrayList<>(this.errors); - this.errors.clear(); - return importErrors; - } - - private void importDerivedFiles(ExternalResults results) { - FileManager fileManager = Case.getCurrentCase().getServices().getFileManager(); - for (ExternalResults.DerivedFile fileData : results.getDerivedFiles()) { - String localPath = fileData.getLocalPath(); - try { - File localFile = new File(localPath); - if (localFile.exists()) { - String relativePath = this.getPathRelativeToCaseFolder(localPath); - if (!relativePath.isEmpty()) { - String parentFilePath = fileData.getParentPath(); - AbstractFile parentFile = findFileInCaseDatabase(parentFilePath); - if (parentFile != null) { - DerivedFile derivedFile = fileManager.addDerivedFile(localFile.getName(), relativePath, localFile.length(), - 0, 0, 0, 0, // Do not currently have file times for derived files from external processes. - true, parentFile, - "", "", "", "", // Not currently providing derivation info for derived files from external processes. - TskData.EncodingType.NONE); // Don't allow external encoded files - IngestServices.getInstance().fireModuleContentEvent(new ModuleContentEvent(derivedFile)); - } else { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.importDerivedFiles.errMsg1.text", - localPath, parentFilePath); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage)); - } - } - } else { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.importDerivedFiles.errMsg2.text", - localPath); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage)); - } - } catch (TskCoreException ex) { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.importDerivedFiles.errMsg3.text", - localPath); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage, ex); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage, ex)); - } - } - } - - @Messages({"ExternalResultsImporter.indexError.message=Failed to index imported artifact for keyword search."}) - private void importArtifacts(ExternalResults results) { - SleuthkitCase caseDb = Case.getCurrentCase().getSleuthkitCase(); - for (ExternalResults.Artifact artifactData : results.getArtifacts()) { - try { - // Add the artifact to the case database. - int artifactTypeId = caseDb.getArtifactType(artifactData.getType()).getTypeID(); - if (artifactTypeId == -1) { - artifactTypeId = caseDb.addBlackboardArtifactType(artifactData.getType(), artifactData.getType()).getTypeID(); - } - Content sourceFile = findFileInCaseDatabase(artifactData.getSourceFilePath()); - if (sourceFile != null) { - BlackboardArtifact artifact = sourceFile.newArtifact(artifactTypeId); - - // Add the artifact's attributes to the case database. - Collection attributes = new ArrayList<>(); - for (ExternalResults.ArtifactAttribute attributeData : artifactData.getAttributes()) { - BlackboardAttribute.Type attributeType = caseDb.getAttributeType(attributeData.getType()); - if (attributeType == null) { - switch (attributeData.getValueType()) { - case "text": //NON-NLS - attributeType = caseDb.addArtifactAttributeType(attributeData.getType(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.fromLabel("String"), attributeData.getType()); //NON-NLS - break; - case "int32": //NON-NLS - attributeType = caseDb.addArtifactAttributeType(attributeData.getType(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.fromLabel("Integer"), attributeData.getType()); //NON-NLS - break; - case "int64": //NON-NLS - attributeType = caseDb.addArtifactAttributeType(attributeData.getType(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.fromLabel("Long"), attributeData.getType()); //NON-NLS - break; - case "double": //NON-NLS - attributeType = caseDb.addArtifactAttributeType(attributeData.getType(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.fromLabel("Double"), attributeData.getType()); //NON-NLS - break; - case "datetime": //NON-NLS - attributeType = caseDb.addArtifactAttributeType(attributeData.getType(), BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.fromLabel("DateTime"), attributeData.getType()); //NON-NLS - } - } - - switch (attributeData.getValueType()) { - case "text": //NON-NLS - attributes.add(new BlackboardAttribute(attributeType, attributeData.getSourceModule(), attributeData.getValue())); - break; - case "int32": //NON-NLS - int intValue = Integer.parseInt(attributeData.getValue()); - attributes.add(new BlackboardAttribute(attributeType, attributeData.getSourceModule(), intValue)); - break; - case "int64": //NON-NLS - long longValue = Long.parseLong(attributeData.getValue()); - attributes.add(new BlackboardAttribute(attributeType, attributeData.getSourceModule(), longValue)); - break; - case "double": //NON-NLS - double doubleValue = Double.parseDouble(attributeData.getValue()); - attributes.add(new BlackboardAttribute(attributeType, attributeData.getSourceModule(), doubleValue)); - break; - case "datetime": //NON-NLS - long dateTimeValue = Long.parseLong(attributeData.getValue()); - attributes.add(new BlackboardAttribute(attributeType, attributeData.getSourceModule(), dateTimeValue)); - break; - default: - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.importArtifacts.caseErrMsg1.text", - attributeData.getType(), attributeData.getValue(), - artifactData.getType(), artifactData.getSourceFilePath(), - attributeData.getValueType()); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage)); - break; - } - } - artifact.addAttributes(attributes); - - try { - // index the artifact for keyword search - blackboard.indexArtifact(artifact); - } catch (Blackboard.BlackboardException ex) { - logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS - MessageNotifyUtil.Notify.error( - Bundle.ExternalResultsImporter_indexError_message(), artifact.getDisplayName()); - } - - if (standardArtifactTypeIds.contains(artifactTypeId)) { - IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(this.getClass().getSimpleName(), BlackboardArtifact.ARTIFACT_TYPE.fromID(artifactTypeId))); - } - } else { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.importArtifacts.errMsg1.text", - artifactData.getType(), artifactData.getSourceFilePath()); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage)); - } - } catch (TskCoreException | TskDataException ex) { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.importArtifacts.errMsg2.text", - artifactData.getType(), artifactData.getSourceFilePath()); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage, ex); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage, ex)); - } - } - } - - private void importReports(ExternalResults results) { - for (ExternalResults.Report report : results.getReports()) { - String reportPath = report.getLocalPath(); - try { - File reportFile = new File(reportPath); - if (reportFile.exists()) { - Case.getCurrentCase().addReport(reportPath, report.getSourceModuleName(), report.getReportName()); - } else { - String errorMessage = NbBundle.getMessage(this.getClass(), "ExternalResultsImporter.importReports.errMsg1.text", reportPath); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage)); - } - } catch (TskCoreException ex) { - String errorMessage = NbBundle.getMessage(this.getClass(), "ExternalResultsImporter.importReports.errMsg2.text", reportPath); - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage, ex); - this.errors.add(new ErrorInfo(ExternalResultsImporter.class.getName(), errorMessage, ex)); - } - } - } - - private AbstractFile findFileInCaseDatabase(String filePath) throws TskCoreException { - AbstractFile file = null; - // Split the path into the file name and the parent path. - String fileName = filePath; - String parentPath = ""; - int charPos = filePath.lastIndexOf("/"); - if (charPos >= 0) { - fileName = filePath.substring(charPos + 1); - parentPath = filePath.substring(0, charPos + 1); - } - // Find the file. - String condition = "name='" + fileName + "' AND parent_path='" + parentPath + "'"; //NON-NLS - List files = Case.getCurrentCase().getSleuthkitCase().findAllFilesWhere(condition); - if (!files.isEmpty()) { - file = files.get(0); - if (files.size() > 1) { - String errorMessage = NbBundle.getMessage(this.getClass(), "ExternalResultsImporter.findFileInCaseDatabase.errMsg1.text", filePath); - this.recordError(errorMessage); - } - } - return file; - } - - private String getPathRelativeToCaseFolder(String localPath) { - String relativePath = ""; - String caseDirectoryPath = Case.getCurrentCase().getCaseDirectory(); - Path path = Paths.get(localPath); - if (path.isAbsolute()) { - Path pathBase = Paths.get(caseDirectoryPath); - try { - Path pathRelative = pathBase.relativize(path); - relativePath = pathRelative.toString(); - } catch (IllegalArgumentException ex) { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.getPathRelativeToCaseFolder.errMsg1.text", - localPath, caseDirectoryPath); - this.recordError(errorMessage, ex); - } - } else { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsImporter.getPathRelativeToCaseFolder.errMsg2.text", - localPath, caseDirectoryPath); - this.recordError(errorMessage); - } - return relativePath; - } - - private void recordError(String errorMessage) { - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage); - this.errors.add(new ErrorInfo(this.getClass().getName(), errorMessage)); - } - - private void recordError(String errorMessage, Exception ex) { - ExternalResultsImporter.logger.log(Level.SEVERE, errorMessage, ex); - this.errors.add(new ErrorInfo(this.getClass().getName(), errorMessage)); - } -} diff --git a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsParser.java b/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsParser.java deleted file mode 100755 index 1b76aa83cf..0000000000 --- a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsParser.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.externalresults; - -import java.util.List; -import org.sleuthkit.autopsy.coreutils.ErrorInfo; - -/** - * Interface for parsers that convert some representation of results data (e.g., - * artifacts, derived files, reports) generated by a process external to Autopsy - * into a form ready for import into Autopsy. - */ -public interface ExternalResultsParser { - - /** - * Converts some representation of results data generated by a process - * external to Autopsy and supplied to the parser via its constructor into a - * form ready for import into Autopsy. - * - * @return External results data in a form ready for import into Autopsy. - */ - ExternalResults parse(); - - /** - * Gets error information describing any errors encountered while parsing - * the input results representation. - * - * @return A collection of error messages, possibly empty. The error - * messages are already logged but are provided to allow the caller - * to provide additional user feedback via the Autopsy user - * interface. - */ - List getErrorInfo(); -} diff --git a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsXMLParser.java b/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsXMLParser.java deleted file mode 100755 index 0c33e87ca7..0000000000 --- a/Core/src/org/sleuthkit/autopsy/externalresults/ExternalResultsXMLParser.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011-2016 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.externalresults; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.ErrorInfo; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.XMLUtil; -import org.sleuthkit.datamodel.Content; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -/** - * Parses an XML representation of of results data (e.g., artifacts, derived - * files, reports) generated by a process external to Autopsy. - */ -public final class ExternalResultsXMLParser implements ExternalResultsParser { - - private static final Logger logger = Logger.getLogger(ExternalResultsXMLParser.class.getName()); - private static final String XSD_FILE = "autopsy_external_results.xsd"; //NON-NLS - private final Content dataSource; - private final String resultsFilePath; - private ExternalResults resultsData; - private List errors = new ArrayList<>(); - - /** - * Tag names for an external results XML file. - */ - public enum TagNames { - - ROOT_ELEM("autopsy_results"), //NON-NLS - DERIVED_FILES_LIST_ELEM("derived_files"), //NON-NLS - DERIVED_FILE_ELEM("derived_file"), //NON-NLS - LOCAL_PATH_ELEM("local_path"), //NON-NLS - PARENT_FILE_ELEM("parent_file"), //NON-NLS - ARTIFACTS_LIST_ELEM("artifacts"), //NON-NLS - ARTIFACT_ELEM("artifact"), //NON-NLS - SOURCE_FILE_ELEM("source_file"), //NON-NLS - ATTRIBUTE_ELEM("attribute"), //NON-NLS - VALUE_ELEM("value"), //NON-NLS - SOURCE_MODULE_ELEM("source_module"), //NON-NLS - REPORTS_LIST_ELEM("reports"), //NON-NLS - REPORT_ELEM("report"), //NON-NLS - REPORT_NAME_ELEM("report_name"); //NON-NLS - private final String text; - - private TagNames(final String text) { - this.text = text; - } - - @Override - public String toString() { - return this.text; - } - } - - /** - * Attribute names for an external results XML file. - */ - public enum AttributeNames { - - TYPE_ATTR("type"); //NON-NLS - private final String text; - - private AttributeNames(final String text) { - this.text = text; - } - - @Override - public String toString() { - return this.text; - } - } - - /** - * Attribute values for an external results XML file. - */ - public enum AttributeValues { - - VALUE_TYPE_TEXT("text"), //NON-NLS - VALUE_TYPE_INT32("int32"), //NON-NLS - VALUE_TYPE_INT64("int64"), //NON-NLS - VALUE_TYPE_DOUBLE("double"), //NON-NLS - VALUE_TYPE_DATETIME("datetime"); //NON-NLS - private final String text; - - private AttributeValues(final String text) { - this.text = text; - } - - @Override - public String toString() { - return this.text; - } - } - - /** - * Constructor. - * - * @param dataSource The data source for the results. - * @param resultsFilePath Full path of the results file to be parsed. - */ - public ExternalResultsXMLParser(Content dataSource, String resultsFilePath) { - this.dataSource = dataSource; - this.resultsFilePath = resultsFilePath; - } - - @Override - public ExternalResults parse() { - this.errors.clear(); - this.resultsData = new ExternalResults(dataSource); - try { - // Note that XMLUtil.loadDoc() logs a warning if the file does not - // conform to the XSD, but still returns a Document object. Until - // this behavior is improved, validation is still required. If - // XMLUtil.loadDoc() does return null, it failed to load the - // document and it logged the error. - final Document doc = XMLUtil.loadDoc(ExternalResultsXMLParser.class, this.resultsFilePath, XSD_FILE); - if (doc != null) { - final Element rootElem = doc.getDocumentElement(); - if (rootElem != null && rootElem.getNodeName().equals(TagNames.ROOT_ELEM.toString())) { - parseDerivedFiles(rootElem); - parseArtifacts(rootElem); - parseReports(rootElem); - } else { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsXMLParser.parse.errMsg1.text", - TagNames.ROOT_ELEM.toString(), this.resultsFilePath); - recordError(errorMessage); - } - } - } catch (Exception ex) { - String errorMessage = NbBundle.getMessage(this.getClass(), "ExternalResultsXMLParser.parse.errMsg2.text", this.resultsFilePath); - recordError(errorMessage, ex); - } - return this.resultsData; - } - - @Override - public List getErrorInfo() { - return new ArrayList<>(this.errors); - } - - private void parseDerivedFiles(Element rootElement) { - // Get the derived file lists. - NodeList derivedFilesListNodes = rootElement.getElementsByTagName(TagNames.DERIVED_FILES_LIST_ELEM.toString()); - for (int i = 0; i < derivedFilesListNodes.getLength(); ++i) { - Element derivedFilesListElem = (Element) derivedFilesListNodes.item(i); - // Get the derived files. - NodeList derivedFileNodes = derivedFilesListElem.getElementsByTagName(TagNames.DERIVED_FILE_ELEM.toString()); - for (int j = 0; j < derivedFileNodes.getLength(); ++j) { - Element derivedFileElem = (Element) derivedFileNodes.item(j); - // Get the local path of the derived file. - String path = getChildElementContent(derivedFileElem, TagNames.LOCAL_PATH_ELEM.toString(), true); - if (path.isEmpty()) { - continue; - } - // Get the parent file of the derived file. - String parentFile = getChildElementContent((Element) derivedFileNodes.item(j), TagNames.PARENT_FILE_ELEM.toString(), true); - if (parentFile.isEmpty()) { - continue; - } - this.resultsData.addDerivedFile(path, parentFile); - } - } - } - - private void parseArtifacts(final Element root) { - // Get the artifact lists. - NodeList artifactsListNodes = root.getElementsByTagName(TagNames.ARTIFACTS_LIST_ELEM.toString()); - for (int i = 0; i < artifactsListNodes.getLength(); ++i) { - Element artifactsListElem = (Element) artifactsListNodes.item(i); - // Get the artifacts. - NodeList artifactNodes = artifactsListElem.getElementsByTagName(TagNames.ARTIFACT_ELEM.toString()); - for (int j = 0; j < artifactNodes.getLength(); ++j) { - Element artifactElem = (Element) artifactNodes.item(j); - // Get the artifact type. - final String type = getElementAttributeValue(artifactElem, AttributeNames.TYPE_ATTR.toString()); - if (!type.isEmpty()) { - // Get the source file of the artifact and the attributes, - // if any. - final String sourceFilePath = this.getChildElementContent(artifactElem, TagNames.SOURCE_FILE_ELEM.toString(), true); - if (!sourceFilePath.isEmpty()) { - ExternalResults.Artifact artifact = this.resultsData.addArtifact(type, sourceFilePath); - parseArtifactAttributes(artifactElem, artifact); - } - } - } - } - } - - private void parseArtifactAttributes(final Element artifactElem, ExternalResults.Artifact artifact) { - // Get the artifact attributes. - NodeList attributeNodesList = artifactElem.getElementsByTagName(TagNames.ATTRIBUTE_ELEM.toString()); - for (int i = 0; i < attributeNodesList.getLength(); ++i) { - Element attributeElem = (Element) attributeNodesList.item(i); - final String type = getElementAttributeValue(attributeElem, AttributeNames.TYPE_ATTR.toString()); - if (type.isEmpty()) { - continue; - } - // Get the value of the artifact attribute. - Element valueElem = this.getChildElement(attributeElem, TagNames.VALUE_ELEM.toString()); - if (valueElem == null) { - continue; - } - final String value = valueElem.getTextContent(); - if (value.isEmpty()) { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsXMLParser.parseArtifactAttributes.errMsg1.text", - TagNames.VALUE_ELEM.toString(), this.resultsFilePath); - recordError(errorMessage); - continue; - } - // Get the value type. - String valueType = parseArtifactAttributeValueType(valueElem); - if (valueType.isEmpty()) { - continue; - } - // Get the optional source module. - String sourceModule = this.getChildElementContent(attributeElem, TagNames.SOURCE_MODULE_ELEM.toString(), false); - // Add the attribute to the artifact. - artifact.addAttribute(type, value, valueType, sourceModule); - } - } - - private String parseArtifactAttributeValueType(Element valueElem) { - String valueType = valueElem.getAttribute(AttributeNames.TYPE_ATTR.toString()); - if (valueType.isEmpty()) { - // Default to text. - valueType = AttributeValues.VALUE_TYPE_TEXT.toString(); - } else if (!valueType.equals(AttributeValues.VALUE_TYPE_TEXT.toString()) - && !valueType.equals(AttributeValues.VALUE_TYPE_DOUBLE.toString()) - && !valueType.equals(AttributeValues.VALUE_TYPE_INT32.toString()) - && !valueType.equals(AttributeValues.VALUE_TYPE_INT64.toString()) - && !valueType.equals(AttributeValues.VALUE_TYPE_DATETIME.toString())) { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsXMLParser.parseAttributeValueType.errMsg1.text", - valueType, - AttributeNames.TYPE_ATTR.toString(), - TagNames.VALUE_ELEM.toString()); - this.recordError(errorMessage); - valueType = ""; - } - return valueType; - } - - private void parseReports(Element root) { - // Get the report lists. - NodeList reportsListNodes = root.getElementsByTagName(TagNames.REPORTS_LIST_ELEM.toString()); - for (int i = 0; i < reportsListNodes.getLength(); ++i) { - Element reportsListElem = (Element) reportsListNodes.item(i); - // Get the reports. - NodeList reportNodes = reportsListElem.getElementsByTagName(TagNames.REPORT_ELEM.toString()); - for (int j = 0; j < reportNodes.getLength(); ++j) { - Element reportElem = (Element) reportNodes.item(j); - // Get the local path. - String path = getChildElementContent(reportElem, TagNames.LOCAL_PATH_ELEM.toString(), true); - if (path.isEmpty()) { - continue; - } - // Get the source module. - String sourceModule = getChildElementContent(reportElem, TagNames.SOURCE_MODULE_ELEM.toString(), true); - if (path.isEmpty()) { - continue; - } - // Get the optional report name. - String reportName = getChildElementContent(reportElem, TagNames.REPORT_NAME_ELEM.toString(), false); - this.resultsData.addReport(path, sourceModule, reportName); - } - } - } - - private String getElementAttributeValue(Element element, String attributeName) { - final String attributeValue = element.getAttribute(attributeName); - if (attributeValue.isEmpty()) { - logger.log(Level.SEVERE, "Found {0} element missing {1} attribute in {2}", new Object[]{ //NON-NLS - element.getTagName(), - attributeName, - this.resultsFilePath}); - } - return attributeValue; - } - - private String getChildElementContent(Element parentElement, String childElementTagName, boolean required) { - String content = ""; - Element childElement = this.getChildElement(parentElement, childElementTagName); - if (childElement != null) { - content = childElement.getTextContent(); - if (content.isEmpty()) { - String errorMessage = NbBundle.getMessage(this.getClass(), - "ExternalResultsXMLParser.getChildElementContent.errMsg1.text", - parentElement.getTagName(), - childElementTagName, - this.resultsFilePath); - this.recordError(errorMessage); - } - } else if (required) { - String errorMessage = NbBundle.getMessage(this.getClass(), "ExternalResultsXMLParser.getChildElementContent.errMsg2.text", - parentElement.getTagName(), - childElementTagName, - this.resultsFilePath); - this.recordError(errorMessage); - } - return content; - } - - private Element getChildElement(Element parentElement, String childElementTagName) { - Element childElem = null; - NodeList childNodes = parentElement.getElementsByTagName(childElementTagName); - if (childNodes.getLength() > 0) { - childElem = (Element) childNodes.item(0); - if (childNodes.getLength() > 1) { - String errorMessage = NbBundle.getMessage(this.getClass(), "ExternalResultsXMLParser.getChildElement.errMsg1.text", - childElementTagName, - parentElement.getTagName(), - this.resultsFilePath); - this.recordError(errorMessage); - } - } - return childElem; - } - - private void recordError(String errorMessage) { - ExternalResultsXMLParser.logger.log(Level.SEVERE, errorMessage); - this.errors.add(new ErrorInfo(this.getClass().getSimpleName(), errorMessage)); - } - - private void recordError(String errorMessage, Exception ex) { - ExternalResultsXMLParser.logger.log(Level.SEVERE, errorMessage, ex); - this.errors.add(new ErrorInfo(this.getClass().getSimpleName(), errorMessage, ex)); - } -} diff --git a/Core/src/org/sleuthkit/autopsy/externalresults/autopsy_external_results.xsd b/Core/src/org/sleuthkit/autopsy/externalresults/autopsy_external_results.xsd deleted file mode 100755 index 1dfd931e47..0000000000 --- a/Core/src/org/sleuthkit/autopsy/externalresults/autopsy_external_results.xsd +++ /dev/null @@ -1,70 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 1d72f2c7f2da5c0d59df4db5c661c2e0768baa9f Mon Sep 17 00:00:00 2001 From: "U-BASIS\\zhaohui" Date: Fri, 13 Oct 2017 12:15:22 -0400 Subject: [PATCH 15/30] NoJira: Remove the obsolete sample --- .../SampleExecutableIngestModuleFactory.java | 79 ------------------- .../ingest/IngestModuleFactoryLoader.java | 5 +- 2 files changed, 1 insertion(+), 83 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/examples/SampleExecutableIngestModuleFactory.java diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableIngestModuleFactory.java deleted file mode 100755 index 390aee2a33..0000000000 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableIngestModuleFactory.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Sample ingest module factory in the public domain. - * Feel free to use this as a template for your inget module factories. - * - * Contact: Brian Carrier [carrier sleuthkit [dot] org] - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -package org.sleuthkit.autopsy.examples; - -import org.openide.util.NbBundle; -import org.openide.util.lookup.ServiceProvider; -import static org.sleuthkit.autopsy.examples.SampleIngestModuleFactory.getModuleName; -import org.sleuthkit.autopsy.ingest.DataSourceIngestModule; -import org.sleuthkit.autopsy.ingest.IngestModuleFactory; -import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; - -/** - * A factory that creates sample executable data source ingest modules. - */ -@ServiceProvider(service = IngestModuleFactory.class) // Sample is discarded at runtime -public class SampleExecutableIngestModuleFactory extends IngestModuleFactoryAdapter { - - private static final String VERSION_NUMBER = "1.0.0"; - - // This class method allows the ingest module instances created by this - // factory to use the same display name that is provided to the Autopsy - // ingest framework by the factory. - static String getModuleName() { - return NbBundle.getMessage(SampleIngestModuleFactory.class, "SampleExecutableIngestModuleFactory.moduleName"); - } - - @Override - public String getModuleDisplayName() { - return getModuleName(); - } - - @Override - public String getModuleDescription() { - return NbBundle.getMessage(SampleIngestModuleFactory.class, "SampleExecutableIngestModuleFactory.moduleDescription"); - } - - @Override - public String getModuleVersionNumber() { - return VERSION_NUMBER; - } - - @Override - public boolean isDataSourceIngestModuleFactory() { - return true; - } - - @Override - public DataSourceIngestModule createDataSourceIngestModule(IngestModuleIngestJobSettings ingestOptions) { - return new SampleExecutableDataSourceIngestModule(); - } -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java index e338b79805..3d2c66deb9 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java @@ -29,7 +29,6 @@ import org.openide.NotifyDescriptor; import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.examples.SampleExecutableIngestModuleFactory; import org.sleuthkit.autopsy.examples.SampleIngestModuleFactory; import org.sleuthkit.autopsy.modules.e01verify.E01VerifierModuleFactory; import org.sleuthkit.autopsy.modules.exif.ExifParserModuleFactory; @@ -48,7 +47,6 @@ final class IngestModuleFactoryLoader { private static final Logger logger = Logger.getLogger(IngestModuleFactoryLoader.class.getName()); private static final String SAMPLE_MODULE_FACTORY_CLASS_NAME = SampleIngestModuleFactory.class.getCanonicalName(); - private static final String SAMPLE_EXECUTABLE_MODULE_FACTORY_CLASS_NAME = SampleExecutableIngestModuleFactory.class.getCanonicalName(); private static final ArrayList coreModuleOrdering = new ArrayList() { { // The ordering of the core ingest module factories implemented @@ -141,8 +139,7 @@ final class IngestModuleFactoryLoader { private static void addFactory(IngestModuleFactory factory, HashSet moduleDisplayNames, HashMap javaFactoriesByClass) { // Ignore the sample ingest module factories implemented in Java. String className = factory.getClass().getCanonicalName(); - if (className.equals(IngestModuleFactoryLoader.SAMPLE_MODULE_FACTORY_CLASS_NAME) - || className.equals(IngestModuleFactoryLoader.SAMPLE_EXECUTABLE_MODULE_FACTORY_CLASS_NAME)) { + if (className.equals(IngestModuleFactoryLoader.SAMPLE_MODULE_FACTORY_CLASS_NAME)) { return; } From 8ac6e71afeee19d5503ecac07bec2d7de9a7b1c0 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sat, 14 Oct 2017 07:52:36 -0400 Subject: [PATCH 16/30] Clean up and better document QueryResults class --- .../keywordsearch/HighlightedText.java | 6 +- .../KeywordSearchResultFactory.java | 2 +- .../autopsy/keywordsearch/QueryResults.java | 254 ++++++++++++------ .../autopsy/keywordsearch/SearchRunner.java | 2 +- 4 files changed, 170 insertions(+), 94 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java index b8bb93b240..6836483b30 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java @@ -104,13 +104,13 @@ class HighlightedText implements IndexedText { /** * This constructor is used when keyword hits are accessed from the ad-hoc * search results. In that case we have the entire QueryResults object and - * need to arrange the paging. + need to arrange the paging. * * @param objectId The objectID of the content whose text will be * highlighted. * @param QueryResults The QueryResults for the ad-hoc search from whose - * results a selection was made leading to this - * HighlightedText. + results a selection was made leading to this + HighlightedText. */ HighlightedText(long objectId, QueryResults hits) { this.objectId = objectId; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java index ac7cfae557..a48b40e756 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java @@ -331,7 +331,7 @@ class KeywordSearchResultFactory extends ChildFactory { final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr; try { progress = ProgressHandle.createHandle(NbBundle.getMessage(this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), () -> BlackboardResultWriter.this.cancel(true)); - hits.writeAllHitsToBlackBoard(progress, null, this, false); + hits.process(progress, null, this, false); } finally { finalizeWorker(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 48fdb38f10..516b96eaab 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -45,195 +45,272 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** - * Stores the results from running a Solr query (which could contain multiple - * keywords). - * + * Stores and processes the results of a keyword search query. Processing + * includes posting keyword hit artifacts to the blackboard, sending messages + * about the search hits to the ingest inbox, and publishing an event to notify + * subscribers of the blackboard posts. */ class QueryResults { private static final Logger logger = Logger.getLogger(QueryResults.class.getName()); private static final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); - /** - * The query that generated the results. - */ - private final KeywordSearchQuery keywordSearchQuery; - - /** - * A map of keywords to keyword hits. - */ + private final KeywordSearchQuery query; private final Map> results = new HashMap<>(); + /** + * Constructs a object that stores and processes the results of a keyword + * search query. Processing includes posting keyword hit artifacts to the + * blackboard, sending messages about the search hits to the ingest inbox, + * and publishing an event to notify subscribers of the blackboard posts. + * + * @param query The query. + */ QueryResults(KeywordSearchQuery query) { - this.keywordSearchQuery = query; + this.query = query; } + /** + * Gets the keyword search query that generated the results stored in this + * object. + * + * @return The query. + */ + KeywordSearchQuery getQuery() { + return query; + } + + /** + * Adds the keyword hits for a keyword to the hits that are stored in this + * object. All calls to this method MUST be completed before calling the + * process method. + * + * @param keyword The keyword, + * @param hits The hits. + */ void addResult(Keyword keyword, List hits) { results.put(keyword, hits); } - KeywordSearchQuery getQuery() { - return keywordSearchQuery; - } - + /** + * Gets the keyword hits stored in this object for a given keyword. + * + * @param keyword The keyword. + * + * @return The keyword hits. + */ List getResults(Keyword keyword) { return results.get(keyword); } + /** + * Gets the set of unique keywords for which keyword hits have been stored + * in this object. + * + * @return + */ Set getKeywords() { return results.keySet(); } /** - * Writes the keyword hits encapsulated in this query result to the - * blackboard. Makes one artifact per keyword per searched object (file or - * artifact), i.e., if a keyword is found several times in the object, only - * one artifact is created. + * Processes the keyword hits stored in this object by osting keyword hit + * artifacts to the blackboard, sending messages about the search hits to + * the ingest inbox, and publishing an event to notify subscribers of the + * blackboard posts. * - * @param progress Can be null. - * @param subProgress Can be null. - * @param worker The Swing worker that is writing the hits, needed to - * support cancellation. - * @param notifyInbox Whether or not write a message to the ingest messages - * inbox. + * Makes one artifact per keyword per searched text source object (file or + * artifact), i.e., if a keyword is found several times in the text + * extracted from the source object, only one artifact is created. + * + * This method ASSUMES that the processing is being done using a SwingWorker + * that should be checked for task cancellation. + * + * All calls to the addResult method MUST be completed before calling this + * method. + * + * @param progress A progress indicator that reports the number of + * keywords processed. Can be null. + * @param subProgress A progress contributor that reports the keyword + * currently being processed. Can be null. + * @param worker The SwingWorker that is being used to do the + * processing, will be checked for task cancellation + * before processing each keyword. + * @param notifyInbox Whether or not to write a message to the ingest + * messages inbox if there is a keyword hit in the text + * exrtacted from the text source object. * - * @return The artifacts that were created. */ - Collection writeAllHitsToBlackBoard(ProgressHandle progress, ProgressContributor subProgress, SwingWorker worker, boolean notifyInbox) { - final Collection newArtifacts = new ArrayList<>(); - if (progress != null) { + void process(ProgressHandle progress, ProgressContributor subProgress, SwingWorker worker, boolean notifyInbox) { + /* + * Initialize the progress indicator to the number of keywords that will + * be processed. + */ + if (null != progress) { progress.start(getKeywords().size()); } - int unitProgress = 0; + /* + * Process the keyword hits for each keyword. + */ + int keywordsProcessed = 0; + final Collection hitArtifacts = new ArrayList<>(); for (final Keyword keyword : getKeywords()) { + /* + * Cancellation check. + */ if (worker.isCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS + logger.log(Level.INFO, "Processing cancelled, exiting before processing search term {0}", keyword.getSearchTerm()); //NON-NLS break; } - // Update progress object(s), if any + /* + * Update the progress indicator and the show the current keyword + * via the progress contributor. + */ if (progress != null) { - progress.progress(keyword.toString(), unitProgress); + progress.progress(keyword.toString(), keywordsProcessed); } if (subProgress != null) { String hitDisplayStr = keyword.getSearchTerm(); if (hitDisplayStr.length() > 50) { hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; } - subProgress.progress(keywordSearchQuery.getKeywordList().getName() + ": " + hitDisplayStr, unitProgress); + subProgress.progress(query.getKeywordList().getName() + ": " + hitDisplayStr, keywordsProcessed); } - for (KeywordHit hit : getOneHitPerObject(keyword)) { - String termString = keyword.getSearchTerm(); + /* + * Reduce the hits for this keyword to one hit per text source + * object so that only one hit artifact is generated per text source + * object, no matter how many times the keyword was actually found. + */ + for (KeywordHit hit : getOneHitPerTextSourceObject(keyword)) { + /* + * Get a snippet (preview) for the hit. Regex queries always + * have snippets made from the content_str pulled back from Solr + * for executing the search. Other types of queries may or may + * not have snippets yet. + */ String snippet = hit.getSnippet(); if (StringUtils.isBlank(snippet)) { - final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(termString); + final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(keyword.getSearchTerm()); try { - /* - * this doesn't work for regex queries... But that is - * okay because regex queries always have snippets made - * from the content_str field we pull back from Solr - */ - snippet = LuceneQuery.querySnippet(snippetQuery, hit.getSolrObjectId(), hit.getChunkId(), !keywordSearchQuery.isLiteral(), true); + snippet = LuceneQuery.querySnippet(snippetQuery, hit.getSolrObjectId(), hit.getChunkId(), !query.isLiteral(), true); } catch (NoOpenCoreException e) { - logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); //NON-NLS - //no reason to continue - break; + logger.log(Level.SEVERE, "Solr core closed while executing snippet query " + snippetQuery, e); //NON-NLS + break; // Stop processing. } catch (Exception e) { - logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); //NON-NLS - continue; + logger.log(Level.SEVERE, "Error executing snippet query " + snippetQuery, e); //NON-NLS + continue; // Try processing the next hit. } } + + /* + * Get the content (file or artifact) that is the text source + * for the hit. + */ Content content = null; try { SleuthkitCase tskCase = Case.getCurrentCase().getSleuthkitCase(); content = tskCase.getContentById(hit.getContentID()); } catch (TskCoreException | IllegalStateException tskCoreException) { - logger.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", tskCoreException); //NON-NLS - return null; + logger.log(Level.SEVERE, "Failed to get text source object for ", tskCoreException); //NON-NLS } - BlackboardArtifact writeResult = keywordSearchQuery.writeSingleFileHitsToBlackBoard(content, keyword, hit, snippet, keywordSearchQuery.getKeywordList().getName()); - if (writeResult != null) { - newArtifacts.add(writeResult); + + /* + * Post an artifact for the hit to the blackboard. + */ + BlackboardArtifact artifact = query.writeSingleFileHitsToBlackBoard(content, keyword, hit, snippet, query.getKeywordList().getName()); + if (null == artifact) { + logger.log(Level.SEVERE, "Error posting keyword hit artifact for keyword {0} in {1} to the blackboard", new Object[]{keyword.toString(), content}); //NON-NLS + } + + /* + * Send an ingest inbox message for the hit. + */ + if (null != artifact) { + hitArtifacts.add(artifact); if (notifyInbox) { try { - writeSingleFileInboxMessage(writeResult, content); + writeSingleFileInboxMessage(artifact, content); } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Error posting message to Ingest Inbox", ex); //NON-NLS + logger.log(Level.SEVERE, "Error sending message to ingest messages inbox", ex); //NON-NLS } } - } else { - logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{content, keyword.toString()}); //NON-NLS } } - ++unitProgress; + + ++keywordsProcessed; } - // Update artifact browser - if (!newArtifacts.isEmpty()) { - newArtifacts.stream() - //group artifacts by type + /* + * Publish an event to notify subscribers of the blackboard posts. The + * artifacts are grouped by type, since they may contain both + * TSK_KEYWORD_HIT artifacts and TSK_ACCOUNT artifacts (for credit card + * account number hits). + */ + if (!hitArtifacts.isEmpty()) { + hitArtifacts.stream() + // Group artifacts by type .collect(Collectors.groupingBy(BlackboardArtifact::getArtifactTypeID)) - //for each type send an event + // For each type send an event .forEach((typeID, artifacts) -> IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.fromID(typeID), artifacts))); } - - return newArtifacts; } /** - * Gets the first hit of the keyword. + * Reduce the hits for a given keyword to one hit per text source object so + * that only one hit artifact is generated per text source object, no matter + * how many times the keyword was actually found. * - * @param keyword + * @param keyword The keyword. * - * @return Collection containing KeywordHits with lowest - * SolrObjectID-ChunkID pairs. + * @return Collection The reduced set of keyword hits. */ - private Collection getOneHitPerObject(Keyword keyword) { - HashMap hits = new HashMap<>(); - - // create a list of KeywordHits. KeywordHits with lowest chunkID is added the the list. - for (KeywordHit hit : getResults(keyword)) { + private Collection getOneHitPerTextSourceObject(Keyword keyword) { + /* + * For each Solr document (chunk) for a text source object, return only + * a single keyword hit from the first chunk of text (the one with the + * lowest chunk id). + */ + HashMap< Long, KeywordHit> hits = new HashMap<>(); + getResults(keyword).forEach((hit) -> { if (!hits.containsKey(hit.getSolrObjectId())) { hits.put(hit.getSolrObjectId(), hit); } else if (hit.getChunkId() < hits.get(hit.getSolrObjectId()).getChunkId()) { hits.put(hit.getSolrObjectId(), hit); } - } + }); return hits.values(); } /** - * Generate and post an ingest inbox message for the given keyword in the - * given content. + * Send an ingest inbox message indicating that there was a keyword hit in + * the given text source object. * - * @param artifact The keyword hit artifact. - * @param hitContent The content that the hit is in. + * @param artifact The keyword hit artifact for the hit. + * @param hitContent The text source object. * - * @throws TskCoreException If there is a problem generating or posting the + * @throws TskCoreException If there is a problem generating or send the * inbox message. */ private void writeSingleFileInboxMessage(BlackboardArtifact artifact, Content hitContent) throws TskCoreException { - StringBuilder subjectSb = new StringBuilder(); - StringBuilder detailsSb = new StringBuilder(); - - if (!keywordSearchQuery.isLiteral()) { + StringBuilder subjectSb = new StringBuilder(1024); + if (!query.isLiteral()) { subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl")); } else { subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl")); } + StringBuilder detailsSb = new StringBuilder(1024); String uniqueKey = null; BlackboardAttribute attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD)); if (attr != null) { final String keyword = attr.getValueString(); subjectSb.append(keyword); uniqueKey = keyword.toLowerCase(); - //details detailsSb.append(""); //NON-NLS - //hit detailsSb.append(""); //NON-NLS detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitThLbl")); detailsSb.append(""); //NON-NLS @@ -270,7 +347,7 @@ class QueryResults { } //regex - if (!keywordSearchQuery.isLiteral()) { + if (!query.isLiteral()) { attr = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP)); if (attr != null) { detailsSb.append(""); //NON-NLS @@ -279,7 +356,6 @@ class QueryResults { detailsSb.append(""); //NON-NLS } } - detailsSb.append("
").append(EscapeUtil.escapeHtml(keyword)).append("
"); //NON-NLS IngestServices.getInstance().postMessage(IngestMessage.createDataMessage(MODULE_NAME, subjectSb.toString(), detailsSb.toString(), uniqueKey, artifact)); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index c9b95a770b..b998294fe4 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -492,7 +492,7 @@ public final class SearchRunner { subProgresses[keywordsSearched].progress(keywordList.getName() + ": " + queryDisplayStr, unitProgress); // Create blackboard artifacts - newResults.writeAllHitsToBlackBoard(null, subProgresses[keywordsSearched], this, keywordList.getIngestMessages()); + newResults.process(null, subProgresses[keywordsSearched], this, keywordList.getIngestMessages()); } //if has results From 32af6d3b4925620bed54d92d21841c40b9c9da8d Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Sat, 14 Oct 2017 08:06:15 -0400 Subject: [PATCH 17/30] Improve documnetation of KeywordSearchQuery method --- .../keywordsearch/KeywordSearchQuery.java | 10 +++---- .../autopsy/keywordsearch/LuceneQuery.java | 30 ++++++++++++++----- .../autopsy/keywordsearch/QueryResults.java | 4 ++- .../autopsy/keywordsearch/RegexQuery.java | 18 ++++++++++- .../keywordsearch/TermsComponentQuery.java | 23 +++++++++++--- 5 files changed, 66 insertions(+), 19 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java index da269e2d2c..4f71b18f5c 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java @@ -101,20 +101,20 @@ interface KeywordSearchQuery { String getEscapedQueryString(); /** - * Converts the keyword hits for a given search term into artifacts. + * Posts a keyword hit artifact to the blackboard for a given keyword hit. * - * @param content The Content object associated with the hit. + * @param content The text source object for the hit. * @param foundKeyword The keyword that was found by the search, this may be * different than the Keyword that was searched if, for * example, it was a RegexQuery. * @param hit The keyword hit. - * @param snippet The document snippet that contains the hit. + * @param snippet A snippet from the text that contains the hit. * @param listName The name of the keyword list that contained the * keyword for which the hit was found. * * - * @return The newly created artifact or Null if there was a problem + * @return The newly created artifact or null if there was a problem * creating it. */ - BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName); + BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 4d3f9cc5b9..e13531da26 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -52,8 +52,8 @@ class LuceneQuery implements KeywordSearchQuery { private static final Logger logger = Logger.getLogger(LuceneQuery.class.getName()); private String keywordStringEscaped; private boolean isEscaped; - private final Keyword originalKeyword ; - private final KeywordList keywordList ; + private final Keyword originalKeyword; + private final KeywordList keywordList; private final List filters = new ArrayList<>(); private String field = null; private static final int MAX_RESULTS_PER_CURSOR_MARK = 512; @@ -70,7 +70,7 @@ class LuceneQuery implements KeywordSearchQuery { LuceneQuery(KeywordList keywordList, Keyword keyword) { this.keywordList = keywordList; this.originalKeyword = keyword; - this.keywordStringEscaped = this.originalKeyword.getSearchTerm(); + this.keywordStringEscaped = this.originalKeyword.getSearchTerm(); } @Override @@ -191,8 +191,24 @@ class LuceneQuery implements KeywordSearchQuery { return StringUtils.isNotBlank(originalKeyword.getSearchTerm()); } + /** + * Posts a keyword hit artifact to the blackboard for a given keyword hit. + * + * @param content The text source object for the hit. + * @param foundKeyword The keyword that was found by the search, this may be + * different than the Keyword that was searched if, for + * example, it was a RegexQuery. + * @param hit The keyword hit. + * @param snippet A snippet from the text that contains the hit. + * @param listName The name of the keyword list that contained the + * keyword for which the hit was found. + * + * + * @return The newly created artifact or null if there was a problem + * creating it. + */ @Override - public BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); Collection attributes = new ArrayList<>(); @@ -225,11 +241,9 @@ class LuceneQuery implements KeywordSearchQuery { } } - hit.getArtifactID().ifPresent(artifactID -> attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) ); - try { bba.addAttributes(attributes); //write out to bb @@ -398,10 +412,10 @@ class LuceneQuery implements KeywordSearchQuery { return EscapeUtil.unEscapeHtml(contentHighlights.get(0)).trim(); } } catch (NoOpenCoreException ex) { - logger.log(Level.SEVERE, "Error executing Lucene Solr Query: " + query +". Solr doc id " + solrObjectId + ", chunkID " + chunkID , ex); //NON-NLS + logger.log(Level.SEVERE, "Error executing Lucene Solr Query: " + query + ". Solr doc id " + solrObjectId + ", chunkID " + chunkID, ex); //NON-NLS throw ex; } catch (KeywordSearchModuleException ex) { - logger.log(Level.SEVERE, "Error executing Lucene Solr Query: " + query +". Solr doc id " + solrObjectId + ", chunkID " + chunkID , ex); //NON-NLS + logger.log(Level.SEVERE, "Error executing Lucene Solr Query: " + query + ". Solr doc id " + solrObjectId + ", chunkID " + chunkID, ex); //NON-NLS return ""; } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 516b96eaab..7037f898be 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -63,6 +63,8 @@ class QueryResults { * blackboard, sending messages about the search hits to the ingest inbox, * and publishing an event to notify subscribers of the blackboard posts. * + * The KeywordSearchQuery is used to do the blackboard posts. + * * @param query The query. */ QueryResults(KeywordSearchQuery query) { @@ -219,7 +221,7 @@ class QueryResults { /* * Post an artifact for the hit to the blackboard. */ - BlackboardArtifact artifact = query.writeSingleFileHitsToBlackBoard(content, keyword, hit, snippet, query.getKeywordList().getName()); + BlackboardArtifact artifact = query.postKeywordHitToBlackboard(content, keyword, hit, snippet, query.getKeywordList().getName()); if (null == artifact) { logger.log(Level.SEVERE, "Error posting keyword hit artifact for keyword {0} in {1} to the blackboard", new Object[]{keyword.toString(), content}); //NON-NLS } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index 392a5e8bc8..0927b68d6c 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -401,8 +401,24 @@ final class RegexQuery implements KeywordSearchQuery { return escapedQuery; } + /** + * Posts a keyword hit artifact to the blackboard for a given keyword hit. + * + * @param content The text source object for the hit. + * @param foundKeyword The keyword that was found by the search, this may be + * different than the Keyword that was searched if, for + * example, it was a RegexQuery. + * @param hit The keyword hit. + * @param snippet A snippet from the text that contains the hit. + * @param listName The name of the keyword list that contained the + * keyword for which the hit was found. + * + * + * @return The newly created artifact or null if there was a problem + * creating it. + */ @Override - public BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); if (content == null) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java index 58087c4590..ab647ec048 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java @@ -79,8 +79,8 @@ final class TermsComponentQuery implements KeywordSearchQuery { * digit is 2 through 6 * */ - static final Pattern CREDIT_CARD_NUM_PATTERN = - Pattern.compile("(?[2-6]([ -]?[0-9]){11,18})"); + static final Pattern CREDIT_CARD_NUM_PATTERN + = Pattern.compile("(?[2-6]([ -]?[0-9]){11,18})"); static final Pattern CREDIT_CARD_TRACK1_PATTERN = Pattern.compile( /* * Track 1 is alphanumeric. @@ -124,7 +124,6 @@ final class TermsComponentQuery implements KeywordSearchQuery { + "?)?)?)?)?)?"); //close nested optional groups //NON-NLS static final BlackboardAttribute.Type KEYWORD_SEARCH_DOCUMENT_ID = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_DOCUMENT_ID); - /** * Constructs an object that implements a regex query that will be performed * as a two step operation. In the first step, the Solr terms component is @@ -325,8 +324,24 @@ final class TermsComponentQuery implements KeywordSearchQuery { return results; } + /** + * Posts a keyword hit artifact to the blackboard for a given keyword hit. + * + * @param content The text source object for the hit. + * @param foundKeyword The keyword that was found by the search, this may be + * different than the Keyword that was searched if, for + * example, it was a RegexQuery. + * @param hit The keyword hit. + * @param snippet A snippet from the text that contains the hit. + * @param listName The name of the keyword list that contained the + * keyword for which the hit was found. + * + * + * @return The newly created artifact or null if there was a problem + * creating it. + */ @Override - public BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { /* * Create either a "plain vanilla" keyword hit artifact with keyword and * regex attributes, or a credit card account artifact with attributes From c17863930d9a19da425c7949ac1f9b6e6c72e874 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 18 Oct 2017 13:41:47 -0400 Subject: [PATCH 18/30] Correct docs for AutoIngestCaseManager --- .../autopsy/experimental/autoingest/AutoIngestCaseManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java index 1ba6ca35ba..d9cd2fd1df 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java @@ -151,7 +151,7 @@ final class AutoIngestCaseManager { /** * Exception type thrown when there is an error completing an auto ingest - * monitor operation. + * case manager operation. */ static final class AutoIngestCaseManagerException extends Exception { From c208e07504b9ef4050d08a1cfd2d40f16b284dfa Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 18 Oct 2017 14:59:38 -0400 Subject: [PATCH 19/30] Modified the handling of logging and exceptions. --- .../autoingest/AutoIngestCaseManager.java | 54 ++++++------------- 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java index d9cd2fd1df..8469ff2315 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestCaseManager.java @@ -22,23 +22,16 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.openide.util.NbBundle; -import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CaseActionException; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.experimental.configuration.AutoIngestUserPreferences; /** * Handles locating and opening cases created by auto ingest. */ final class AutoIngestCaseManager { - private static final Logger LOGGER = Logger.getLogger(AutoIngestCaseManager.class.getName()); private static AutoIngestCaseManager instance; private CoordinationService coordinationService; @@ -47,6 +40,8 @@ final class AutoIngestCaseManager { * Gets the auto ingest case manager. * * @return The auto ingest case manager singleton. + * + * @throws AutoIngestCaseManagerException */ synchronized static AutoIngestCaseManager getInstance() throws AutoIngestCaseManager.AutoIngestCaseManagerException { if (null == instance) { @@ -58,48 +53,30 @@ final class AutoIngestCaseManager { /** * Constructs an object that handles locating and opening cases created by * auto ingest. + * + * @throws AutoIngestCaseManagerException */ private AutoIngestCaseManager() throws AutoIngestCaseManagerException { - try { - init(); - } catch (AutoIngestCaseManager.AutoIngestCaseManagerException ex) { - Logger.getLogger(AutoIngestCaseManager.class.getName()).log(Level.SEVERE, null, ex); - throw ex; - } - } - - /** - * Initialize the coordination service. - * - * @throws org.sleuthkit.autopsy.experimental.autoingest.AutoIngestCaseManager.AutoIngestCaseManagerException - */ - private void init() throws AutoIngestCaseManager.AutoIngestCaseManagerException { try { coordinationService = CoordinationService.getInstance(); } catch (CoordinationServiceException ex) { - throw new AutoIngestCaseManager.AutoIngestCaseManagerException(ex.getMessage(), ex); + throw new AutoIngestCaseManager.AutoIngestCaseManagerException("Failed to get the coordination service.", ex); } } /** * Gets a list of the cases in the top level case folder used by auto * ingest. + * + * @return List of cases. + * + * @throws AutoIngestCaseManagerException */ - @Messages({ - "AutoIngestCaseManager.CasePathsError=An error occurred while trying to retrieve the list of case paths." - }) - List getCases() { + List getCases() throws AutoIngestCaseManagerException { List cases = new ArrayList<>(); - List casePathList; - try { - casePathList = getCasePaths(); - for (Path casePath : casePathList) { - cases.add(new AutoIngestCase(casePath)); - } - } catch (AutoIngestCaseManagerException ex) { - String errorMessage = NbBundle.getMessage(AutoIngestCaseManager.class, "AutoIngestCaseManager.CasePathsError"); - LOGGER.log(Level.SEVERE, errorMessage, ex); - MessageNotifyUtil.Message.error(errorMessage); + List casePathList = getCasePaths(); + for (Path casePath : casePathList) { + cases.add(new AutoIngestCase(casePath)); } return cases; } @@ -109,6 +86,8 @@ final class AutoIngestCaseManager { * case paths. * * @return List of case paths. + * + * @throws AutoIngestCaseManagerException */ private List getCasePaths() throws AutoIngestCaseManagerException { try { @@ -122,7 +101,8 @@ final class AutoIngestCaseManager { String nodeUpperCase = node.toUpperCase(); if(!nodeUpperCase.endsWith("_RESOURCES") && !nodeUpperCase.endsWith("AUTO_INGEST_LOG.TXT")) { /* - * This is not a case resource lock. Collect the path. + * This is not a case resource lock, nor a case auto + * ingest log lock. Collect the path. */ casePathList.add(Paths.get(node)); } From 89f48e8c384e5504ffc515618e9d7897c4368ac5 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Oct 2017 16:23:01 -0400 Subject: [PATCH 20/30] First cut at ArchiveExtractorDataSourceProcessor and ArchiveFilePanel --- .../autoingest/AddDataSourceCallback.java | 19 +- .../ArchiveExtractorDataSourceProcessor.java | 260 ++++++++++++++++ .../autoingest/ArchiveFilePanel.form | 94 ++++++ .../autoingest/ArchiveFilePanel.java | 280 ++++++++++++++++++ .../experimental/autoingest/Bundle.properties | 6 + .../experimental/autoingest/DataSource.java | 19 +- .../IdentifyDataSourceProcessors.java | 19 +- 7 files changed, 688 insertions(+), 9 deletions(-) create mode 100755 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java create mode 100755 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form create mode 100755 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java index f566dffb73..db19fc2fbc 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddDataSourceCallback.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.experimental.autoingest; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java new file mode 100755 index 0000000000..19dbd61f4a --- /dev/null +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java @@ -0,0 +1,260 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.experimental.autoingest; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import javax.swing.JPanel; +import javax.swing.filechooser.FileFilter; +import org.apache.commons.io.FilenameUtils; +import org.openide.util.NbBundle; +import org.openide.util.lookup.ServiceProvider; +import org.openide.util.lookup.ServiceProviders; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.GeneralFilter; +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; + +/** + * A data source processor that handles archive files. Implements the + * DataSourceProcessor service provider interface to allow integration with the + * add data source wizard. It also provides a run method overload to allow it to + * be used independently of the wizard. + */ +@ServiceProviders(value={ + @ServiceProvider(service=DataSourceProcessor.class), + @ServiceProvider(service=AutoIngestDataSourceProcessor.class)} +) +public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, AutoIngestDataSourceProcessor { + + private final static String DATA_SOURCE_TYPE = NbBundle.getMessage(ArchiveExtractorDataSourceProcessor.class, "ArchiveExtractorDataSourceProcessor.dsType.text"); + + private static GeneralFilter zipFilter; + private static List archiveFilters = new ArrayList<>(); + + private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; + + private final ArchiveFilePanel configPanel; + private String deviceId; + private String imagePath; + private boolean setDataSourceOptionsCalled; + + /** + * Constructs an archive data source processor that + * implements the DataSourceProcessor service provider interface to allow + * integration with the add data source wizard. It also provides a run + * method overload to allow it to be used independently of the wizard. + */ + public ArchiveExtractorDataSourceProcessor() { + String[] extensions = ArchiveUtil.getSupportedArchiveTypes(); + zipFilter = new GeneralFilter(Arrays.asList(extensions), ""); + archiveFilters.add(zipFilter); + configPanel = ArchiveFilePanel.createInstance(ArchiveExtractorDataSourceProcessor.class.getName(), archiveFilters); + } + + @Override + public int canProcess(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { + // check whether this is an archive + if (isArchive(dataSourcePath)){ + // return "high confidence" value + return 100; + } + return 0; + } + + @Override + public void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) throws AutoIngestDataSourceProcessorException { + if (isArchive(dataSourcePath)) { + // extract the archive and pass the extracted folder as input + Path extractedDataSourcePath = Paths.get(""); + try { + Case currentCase = Case.getCurrentCase(); + extractedDataSourcePath = extractDataSource(Paths.get(currentCase.getModuleDirectory()), dataSourcePath); + } catch (Exception ex) { + throw new AutoIngestDataSourceProcessorException(NbBundle.getMessage(ArchiveExtractorDataSourceProcessor.class, "ArchiveExtractorDataSourceProcessor.process.exception.text"), ex); + } + run(deviceId, extractedDataSourcePath.toString(), progressMonitor, callBack); + } + } + + @Override + public String getDataSourceType() { + return DATA_SOURCE_TYPE; + } + + /** + * Gets the panel that allows a user to select a data source and do any + * configuration required by the data source. The panel is less than 544 + * pixels wide and less than 173 pixels high. + * + * @return A selection and configuration panel for this data source + * processor. + */ + @Override + public JPanel getPanel() { + configPanel.readSettings(); + configPanel.select(); + return configPanel; + } + + /** + * Indicates whether the settings in the selection and configuration panel + * are valid and complete. + * + * @return True if the settings are valid and complete and the processor is + * ready to have its run method called, false otherwise. + */ + @Override + public boolean isPanelValid() { + return configPanel.validatePanel(); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + @Override + public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + if (!setDataSourceOptionsCalled) { + configPanel.storeSettings(); + deviceId = UUID.randomUUID().toString(); + imagePath = configPanel.getContentPaths(); + } + run(deviceId, imagePath, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the given settings instead of those provided by the + * selection and configuration panel. Returns as soon as the background task + * is started and uses the callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + public void run(String deviceId, String imageFolderPath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + //List imageFilePaths = getImageFilePaths(imageFolderPath); + //addImagesTask = new AddCellebritePhysicalReportTask(deviceId, imageFilePaths, timeZone, progressMonitor, callback); + //new Thread(addImagesTask).start(); + } + + @Override + public void cancel() { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void reset() { + deviceId = null; + imagePath = null; + configPanel.reset(); + setDataSourceOptionsCalled = false; + } + + private static boolean isArchive(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { + String fileName = dataSourcePath.getFileName().toString(); + // check whether it's a zip archive file that can be extracted + return isAcceptedByFiler(new File(fileName), archiveFilters); + } + + private static boolean isAcceptedByFiler(File file, List filters) { + for (FileFilter filter : filters) { + if (filter.accept(file)) { + return true; + } + } + return false; + } + + /** + * Extracts the contents of a ZIP archive submitted as a data source to a + * subdirectory of the auto ingest module output directory. + * + * @throws IOException if there is a problem extracting the data source from + * the archive. + */ + private static Path extractDataSource(Path outputDirectoryPath, Path dataSourcePath) throws IOException { + String dataSourceFileNameNoExt = FilenameUtils.removeExtension(dataSourcePath.getFileName().toString()); + Path destinationFolder = Paths.get(outputDirectoryPath.toString(), + AUTO_INGEST_MODULE_OUTPUT_DIR, + dataSourceFileNameNoExt + "_" + TimeStampUtils.createTimeStamp()); + Files.createDirectories(destinationFolder); + + int BUFFER_SIZE = 524288; // Read/write 500KB at a time + File sourceZipFile = dataSourcePath.toFile(); + ZipFile zipFile; + zipFile = new ZipFile(sourceZipFile, ZipFile.OPEN_READ); + Enumeration zipFileEntries = zipFile.entries(); + try { + while (zipFileEntries.hasMoreElements()) { + ZipEntry entry = zipFileEntries.nextElement(); + String currentEntry = entry.getName(); + File destFile = new File(destinationFolder.toString(), currentEntry); + destFile = new File(destinationFolder.toString(), destFile.getName()); + File destinationParent = destFile.getParentFile(); + destinationParent.mkdirs(); + if (!entry.isDirectory()) { + BufferedInputStream is = new BufferedInputStream(zipFile.getInputStream(entry)); + int currentByte; + byte data[] = new byte[BUFFER_SIZE]; + try (FileOutputStream fos = new FileOutputStream(destFile); BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) { + currentByte = is.read(data, 0, BUFFER_SIZE); + while (currentByte != -1) { + dest.write(data, 0, currentByte); + currentByte = is.read(data, 0, BUFFER_SIZE); + } + } + } + } + } finally { + zipFile.close(); + } + return destinationFolder; + } +} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form new file mode 100755 index 0000000000..af9347df0c --- /dev/null +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.form @@ -0,0 +1,94 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java new file mode 100755 index 0000000000..e368a18c4f --- /dev/null +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java @@ -0,0 +1,280 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.experimental.autoingest; + +import java.io.File; +import java.util.List; +import java.util.logging.Level; +import javax.swing.JFileChooser; +import javax.swing.JPanel; +import javax.swing.event.DocumentEvent; +import javax.swing.event.DocumentListener; +import javax.swing.filechooser.FileFilter; +import org.apache.commons.lang3.StringUtils; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; +import static org.sleuthkit.autopsy.experimental.autoingest.Bundle.*; +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; +import org.sleuthkit.autopsy.coreutils.DriveUtils; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; +import org.sleuthkit.autopsy.coreutils.PathValidator; + +/** + * Panel for adding an archive file which is supported by 7zip library (e.g. + * "zip", "rar", "arj", "7z", "7zip", "gzip, etc). Allows the user to select a + * file. + */ +public class ArchiveFilePanel extends JPanel implements DocumentListener { + + private static final Logger logger = Logger.getLogger(ArchiveFilePanel.class.getName()); + private static final String PROP_LASTIMAGE_PATH = "LBL_LastImage_PATH"; //NON-NLS + + private final JFileChooser fileChooser = new JFileChooser(); + + /** + * Externally supplied name is used to store settings + */ + private final String contextName; + + /** + * Creates new form ArchiveFilePanel + * + * @param context A string context name used to read/store last + * used settings. + * @param fileChooserFilters A list of filters to be used with the + * FileChooser. + */ + private ArchiveFilePanel(String context, List fileChooserFilters) { + this.contextName = context; + initComponents(); + + errorLabel.setVisible(false); + + fileChooser.setDragEnabled(false); + fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + fileChooser.setMultiSelectionEnabled(false); + fileChooserFilters.forEach(fileChooser::addChoosableFileFilter); + if (fileChooserFilters.isEmpty() == false) { + fileChooser.setFileFilter(fileChooserFilters.get(0)); + } + } + + /** + * Creates and returns an instance of a ArchiveFilePanel. + * + * @param context A string context name used to read/store last + * used settings. + * @param fileChooserFilters A list of filters to be used with the + * FileChooser. + * + * @return instance of the ArchiveFilePanel + */ + public static synchronized ArchiveFilePanel createInstance(String context, List fileChooserFilters) { + ArchiveFilePanel instance = new ArchiveFilePanel(context, fileChooserFilters); + // post-constructor initialization of listener support without leaking references of uninitialized objects + instance.pathTextField.getDocument().addDocumentListener(instance); + return instance; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + // //GEN-BEGIN:initComponents + private void initComponents() { + + pathLabel = new javax.swing.JLabel(); + browseButton = new javax.swing.JButton(); + pathTextField = new javax.swing.JTextField(); + errorLabel = new javax.swing.JLabel(); + + setMinimumSize(new java.awt.Dimension(0, 65)); + setPreferredSize(new java.awt.Dimension(403, 65)); + + org.openide.awt.Mnemonics.setLocalizedText(pathLabel, org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.pathLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(browseButton, org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.browseButton.text")); // NOI18N + browseButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + browseButtonActionPerformed(evt); + } + }); + + pathTextField.setText(org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.pathTextField.text")); // NOI18N + + errorLabel.setForeground(new java.awt.Color(255, 0, 0)); + org.openide.awt.Mnemonics.setLocalizedText(errorLabel, org.openide.util.NbBundle.getMessage(ArchiveFilePanel.class, "ArchiveFilePanel.errorLabel.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(pathTextField) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(browseButton) + .addGap(2, 2, 2)) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(pathLabel) + .addComponent(errorLabel)) + .addGap(0, 277, Short.MAX_VALUE)) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(pathLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(browseButton) + .addComponent(pathTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGap(3, 3, 3) + .addComponent(errorLabel) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + ); + }// //GEN-END:initComponents + + private void browseButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_browseButtonActionPerformed + String oldText = getContentPaths(); + // set the current directory of the FileChooser if the ImagePath Field is valid + File currentDir = new File(oldText); + if (currentDir.exists()) { + fileChooser.setCurrentDirectory(currentDir); + } + + if (fileChooser.showOpenDialog(this) == JFileChooser.APPROVE_OPTION) { + String path = fileChooser.getSelectedFile().getPath(); + setContentPath(path); + } + + updateHelper(); + }//GEN-LAST:event_browseButtonActionPerformed + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JButton browseButton; + private javax.swing.JLabel errorLabel; + private javax.swing.JLabel pathLabel; + private javax.swing.JTextField pathTextField; + // End of variables declaration//GEN-END:variables + + /** + * Get the path of the user selected archive. + * + * @return the image path + */ + public String getContentPaths() { + return pathTextField.getText(); + } + + /** + * Set the path of the archive file. + * + * @param s path of the archive file + */ + public void setContentPath(String s) { + pathTextField.setText(s); + } + + public void reset() { + //reset the UI elements to default + pathTextField.setText(null); + } + + /** + * Should we enable the next button of the wizard? + * + * @return true if a proper image has been selected, false otherwise + */ + @NbBundle.Messages("DataSourceOnCDriveError.text=Warning: Path to multi-user data source is on \"C:\" drive") + public boolean validatePanel() { + errorLabel.setVisible(false); + String path = getContentPaths(); + if (StringUtils.isBlank(path)) { + return false; + } + + // display warning if there is one (but don't disable "next" button) + if (false == PathValidator.isValid(path, Case.getCurrentCase().getCaseType())) { + errorLabel.setVisible(true); + errorLabel.setText(Bundle.DataSourceOnCDriveError_text()); + } + + return new File(path).isFile() + || DriveUtils.isPhysicalDrive(path) + || DriveUtils.isPartition(path); + } + + public void storeSettings() { + String archivePathName = getContentPaths(); + if (null != archivePathName) { + String archivePath = archivePathName.substring(0, archivePathName.lastIndexOf(File.separator) + 1); + ModuleSettings.setConfigSetting(contextName, PROP_LASTIMAGE_PATH, archivePath); + } + } + + public void readSettings() { + String lastArchivePath = ModuleSettings.getConfigSetting(contextName, PROP_LASTIMAGE_PATH); + if (StringUtils.isNotBlank(lastArchivePath)) { + setContentPath(lastArchivePath); + } + } + + @Override + public void insertUpdate(DocumentEvent e) { + updateHelper(); + } + + @Override + public void removeUpdate(DocumentEvent e) { + updateHelper(); + } + + @Override + public void changedUpdate(DocumentEvent e) { + updateHelper(); + } + + /** + * Update functions are called by the pathTextField which has this set as + * it's DocumentEventListener. Each update function fires a property change + * to be caught by the parent panel. + * + */ + @NbBundle.Messages({"ArchiveFilePanel.moduleErr=Module Error", + "ArchiveFilePanel.moduleErr.msg=A module caused an error listening to ArchiveFilePanel updates." + + " See log to determine which module. Some data could be incomplete.\n"}) + private void updateHelper() { + try { + firePropertyChange(DataSourceProcessor.DSP_PANEL_EVENT.UPDATE_UI.toString(), false, true); + } catch (Exception e) { + logger.log(Level.SEVERE, "ArchiveFilePanel listener threw exception", e); //NON-NLS + MessageNotifyUtil.Notify.error(ArchiveFilePanel_moduleErr(), ArchiveFilePanel_moduleErr_msg()); + } + } + + /** + * Set the focus to the pathTextField. + */ + public void select() { + pathTextField.requestFocusInWindow(); + } +} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties index 422c7a56c4..ec32087dbf 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties @@ -247,3 +247,9 @@ AutoIngestDashboard.prioritizeJobButton.toolTipText=Move the selected job to the AutoIngestDashboard.prioritizeJobButton.text=Prioritize &Job AutoIngestDashboard.prioritizeCaseButton.toolTipText=Move all images associated with a case to top of Pending queue. AutoIngestDashboard.prioritizeCaseButton.text=Prioritize &Case +ArchiveExtractorDataSourceProcessor.dsType.text=Archive file +ArchiveExtractorDataSourceProcessor.process.exception.text=Exception while trying to extract archive +ArchiveFilePanel.pathLabel.text=Browse for an archive file: +ArchiveFilePanel.browseButton.text=Browse +ArchiveFilePanel.pathTextField.text= +ArchiveFilePanel.errorLabel.text=Error Label \ No newline at end of file diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java index b655bfe0ae..db9d3d5ad9 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSource.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.experimental.autoingest; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java index d6673ad580..b566d99aac 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.experimental.autoingest; From ab4f586f3213c5823f1841e7bad6f7bb3889a490 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Oct 2017 16:40:35 -0400 Subject: [PATCH 21/30] Created dummy AddArchiveTask --- .../autoingest/AddArchiveTask.java | 84 +++++++++++++++++++ .../ArchiveExtractorDataSourceProcessor.java | 38 ++++++--- .../autoingest/AutoIngestManager.java | 5 -- 3 files changed, 109 insertions(+), 18 deletions(-) create mode 100755 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java new file mode 100755 index 0000000000..e0878d05ee --- /dev/null +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -0,0 +1,84 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.experimental.autoingest; + +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; +import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.coreutils.Logger; + +/* + * A runnable that adds an archive data source to the case database. + */ +public class AddArchiveTask implements Runnable { + + private final Logger logger = Logger.getLogger(AddArchiveTask.class.getName()); + private final String deviceId; + private final String imagePath; + private final DataSourceProcessorProgressMonitor progressMonitor; + private final DataSourceProcessorCallback callback; + private boolean criticalErrorOccurred; + + /* + * The cancellation requested flag and SleuthKit add image process are + * guarded by a monitor (called a lock here to avoid confusion with the + * progress monitor) to synchronize cancelling the process (setting the flag + * and calling its stop method) and calling either its commit or revert + * method. The built-in monitor of the add image process can't be used for + * this because it is already used to synchronize its run (init part), + * commit, revert, and currentDirectory methods. + * + * TODO (AUT-2021): Merge SleuthkitJNI.AddImageProcess and AddImageTask + */ + private final Object tskAddImageProcessLock; + + /** + * Constructs a runnable task that adds an image to the case database. + * + * @param deviceId An ASCII-printable identifier for the device associated + * with the data source that is intended to be unique across multiple cases + * (e.g., a UUID). + * @param imagePath Path to the image file. + * @param progressMonitor Progress monitor to report progress during + * processing. + * @param callback Callback to call when processing is done. + */ + AddArchiveTask(String deviceId, String imagePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + this.deviceId = deviceId; + this.imagePath = imagePath; + this.callback = callback; + this.progressMonitor = progressMonitor; + tskAddImageProcessLock = new Object(); + } + + /** + * Adds the archive to the case database. + */ + @Override + public void run() { + + } + + + /* + * Attempts to cancel adding the image to the case database. + */ + public void cancelTask() { + + } +} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java index 19dbd61f4a..4512e1d349 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java @@ -61,7 +61,7 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, private final static String DATA_SOURCE_TYPE = NbBundle.getMessage(ArchiveExtractorDataSourceProcessor.class, "ArchiveExtractorDataSourceProcessor.dsType.text"); private static GeneralFilter zipFilter; - private static List archiveFilters = new ArrayList<>(); + private static final List archiveFilters = new ArrayList<>(); private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; @@ -70,6 +70,8 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, private String imagePath; private boolean setDataSourceOptionsCalled; + private AddArchiveTask addArchiveTask; + /** * Constructs an archive data source processor that * implements the DataSourceProcessor service provider interface to allow @@ -171,22 +173,32 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, * is started and uses the callback object to signal task completion and * return results. * - * This method should not be called unless isPanelValid returns true. - * - * @param progressMonitor Progress monitor that will be used by the - * background task to report progress. - * @param callback Callback that will be used by the background task - * to return results. + * @param deviceId An ASCII-printable identifier for the device + * associated with the data source that is + * intended to be unique across multiple cases + * (e.g., a UUID). + * @param imagePath Path to the image file. + * @param progressMonitor Progress monitor for reporting progress + * during processing. + * @param callback Callback to call when processing is done. */ - public void run(String deviceId, String imageFolderPath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - //List imageFilePaths = getImageFilePaths(imageFolderPath); - //addImagesTask = new AddCellebritePhysicalReportTask(deviceId, imageFilePaths, timeZone, progressMonitor, callback); - //new Thread(addImagesTask).start(); - } + public void run(String deviceId, String imagePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + addArchiveTask = new AddArchiveTask(deviceId, imagePath, progressMonitor, callback); + new Thread(addArchiveTask).start(); + } + /** + * Requests cancellation of the background task that adds a data source to + * the case database, after the task is started using the run method. This + * is a "best effort" cancellation, with no guarantees that the case + * database will be unchanged. If cancellation succeeded, the list of new + * data sources returned by the background task will be empty. + */ @Override public void cancel() { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + if (null != addArchiveTask) { + addArchiveTask.cancelTask(); + } } @Override diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index f15cf9f5dc..7ffb67ae05 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -37,7 +37,6 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -59,9 +58,6 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.Immutable; -import javax.annotation.concurrent.ThreadSafe; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; @@ -101,7 +97,6 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; -import org.sleuthkit.datamodel.Content; /** * An auto ingest manager is responsible for processing auto ingest jobs defined From 11b66c1648a4c4fb015bfe41613ae54eabdd1efb Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 19 Oct 2017 16:23:31 -0400 Subject: [PATCH 22/30] Moved archive handling code into ArchiveUtil for code re-use --- .../autoingest/AddArchiveTask.java | 26 +++++++ ....java => ArchiveExtractorDSProcessor.java} | 68 +++---------------- .../experimental/autoingest/ArchiveUtil.java | 34 +++++++++- 3 files changed, 67 insertions(+), 61 deletions(-) rename Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/{ArchiveExtractorDataSourceProcessor.java => ArchiveExtractorDSProcessor.java} (77%) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index e0878d05ee..4d744803c9 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -18,9 +18,15 @@ */ package org.sleuthkit.autopsy.experimental.autoingest; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Content; /* * A runnable that adds an archive data source to the case database. @@ -33,6 +39,8 @@ public class AddArchiveTask implements Runnable { private final DataSourceProcessorProgressMonitor progressMonitor; private final DataSourceProcessorCallback callback; private boolean criticalErrorOccurred; + + private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; /* * The cancellation requested flag and SleuthKit add image process are @@ -71,7 +79,25 @@ public class AddArchiveTask implements Runnable { */ @Override public void run() { + if (!ArchiveUtil.isArchive(Paths.get(imagePath))) { + List errorMessages = new ArrayList<>(); + errorMessages.add("Input data source is not a valid datasource: " + imagePath.toString()); + List newDataSources = new ArrayList<>(); + DataSourceProcessorCallback.DataSourceProcessorResult result; + result = DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; + callback.done(result, errorMessages, newDataSources); + } + // extract the archive and pass the extracted folder as input + Path extractedDataSourcePath = Paths.get(""); + try { + Case currentCase = Case.getCurrentCase(); + //extractedDataSourcePath = extractDataSource(Paths.get(currentCase.getModuleDirectory()), imagePath); + } catch (Exception ex) { + //throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException(NbBundle.getMessage(ArchiveExtractorDSProcessor.class, "ArchiveExtractorDataSourceProcessor.process.exception.text"), ex); + } + + // do processing } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java similarity index 77% rename from Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java rename to Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java index 4512e1d349..6fa8238dc3 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDataSourceProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java @@ -18,29 +18,12 @@ */ package org.sleuthkit.autopsy.experimental.autoingest; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.List; import java.util.UUID; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import javax.swing.JPanel; -import javax.swing.filechooser.FileFilter; -import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.GeneralFilter; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; @@ -56,15 +39,10 @@ import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; @ServiceProvider(service=DataSourceProcessor.class), @ServiceProvider(service=AutoIngestDataSourceProcessor.class)} ) -public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, AutoIngestDataSourceProcessor { - - private final static String DATA_SOURCE_TYPE = NbBundle.getMessage(ArchiveExtractorDataSourceProcessor.class, "ArchiveExtractorDataSourceProcessor.dsType.text"); - - private static GeneralFilter zipFilter; - private static final List archiveFilters = new ArrayList<>(); - - private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; +public class ArchiveExtractorDSProcessor implements DataSourceProcessor, AutoIngestDataSourceProcessor { + private final static String DATA_SOURCE_TYPE = NbBundle.getMessage(ArchiveExtractorDSProcessor.class, "ArchiveExtractorDataSourceProcessor.dsType.text"); + private final ArchiveFilePanel configPanel; private String deviceId; private String imagePath; @@ -78,17 +56,14 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, * integration with the add data source wizard. It also provides a run * method overload to allow it to be used independently of the wizard. */ - public ArchiveExtractorDataSourceProcessor() { - String[] extensions = ArchiveUtil.getSupportedArchiveTypes(); - zipFilter = new GeneralFilter(Arrays.asList(extensions), ""); - archiveFilters.add(zipFilter); - configPanel = ArchiveFilePanel.createInstance(ArchiveExtractorDataSourceProcessor.class.getName(), archiveFilters); + public ArchiveExtractorDSProcessor() { + configPanel = ArchiveFilePanel.createInstance(ArchiveExtractorDSProcessor.class.getName(), ArchiveUtil.getArchiveFilters()); } @Override public int canProcess(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { // check whether this is an archive - if (isArchive(dataSourcePath)){ + if (ArchiveUtil.isArchive(dataSourcePath)){ // return "high confidence" value return 100; } @@ -97,17 +72,7 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, @Override public void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) throws AutoIngestDataSourceProcessorException { - if (isArchive(dataSourcePath)) { - // extract the archive and pass the extracted folder as input - Path extractedDataSourcePath = Paths.get(""); - try { - Case currentCase = Case.getCurrentCase(); - extractedDataSourcePath = extractDataSource(Paths.get(currentCase.getModuleDirectory()), dataSourcePath); - } catch (Exception ex) { - throw new AutoIngestDataSourceProcessorException(NbBundle.getMessage(ArchiveExtractorDataSourceProcessor.class, "ArchiveExtractorDataSourceProcessor.process.exception.text"), ex); - } - run(deviceId, extractedDataSourcePath.toString(), progressMonitor, callBack); - } + run(deviceId, dataSourcePath.toString(), progressMonitor, callBack); } @Override @@ -208,21 +173,6 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, configPanel.reset(); setDataSourceOptionsCalled = false; } - - private static boolean isArchive(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { - String fileName = dataSourcePath.getFileName().toString(); - // check whether it's a zip archive file that can be extracted - return isAcceptedByFiler(new File(fileName), archiveFilters); - } - - private static boolean isAcceptedByFiler(File file, List filters) { - for (FileFilter filter : filters) { - if (filter.accept(file)) { - return true; - } - } - return false; - } /** * Extracts the contents of a ZIP archive submitted as a data source to a @@ -230,7 +180,7 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, * * @throws IOException if there is a problem extracting the data source from * the archive. - */ + private static Path extractDataSource(Path outputDirectoryPath, Path dataSourcePath) throws IOException { String dataSourceFileNameNoExt = FilenameUtils.removeExtension(dataSourcePath.getFileName().toString()); Path destinationFolder = Paths.get(outputDirectoryPath.toString(), @@ -268,5 +218,5 @@ public class ArchiveExtractorDataSourceProcessor implements DataSourceProcessor, zipFile.close(); } return destinationFolder; - } + } */ } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java index 9683deb5c6..b7a2926092 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveUtil.java @@ -26,7 +26,9 @@ import java.io.RandomAccessFile; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import javax.swing.filechooser.FileFilter; import net.sf.sevenzipjbinding.ISequentialOutStream; import net.sf.sevenzipjbinding.ISevenZipInArchive; import net.sf.sevenzipjbinding.SevenZip; @@ -35,18 +37,46 @@ import net.sf.sevenzipjbinding.SevenZipNativeInitializationException; import net.sf.sevenzipjbinding.impl.RandomAccessFileInStream; import net.sf.sevenzipjbinding.simple.ISimpleInArchive; import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.GeneralFilter; /** * Set of utilities that handles archive file extraction. Uses 7zip library. */ final class ArchiveUtil { - static final String[] SUPPORTED_EXTENSIONS = {"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",}; // NON-NLS - + private static final String[] SUPPORTED_EXTENSIONS = {"zip", "rar", "arj", "7z", "7zip", "gzip", "gz", "bzip2", "tar", "tgz",}; // NON-NLS + private static final List ARCHIVE_EXTS = Arrays.asList(".zip", ".rar", ".arj", ".7z", ".7zip", ".gzip", ".gz", ".bzip2", ".tar", ".tgz"); //NON-NLS + @NbBundle.Messages("GeneralFilter.archiveDesc.text=Archive Files (.zip, .rar, .arj, .7z, .7zip, .gzip, .gz, .bzip2, .tar, .tgz)") + private static final String ARCHIVE_DESC = Bundle.GeneralFilter_archiveDesc_text(); + private static final GeneralFilter SEVEN_ZIP_FILTER = new GeneralFilter(ARCHIVE_EXTS, ARCHIVE_DESC); + private static final List ARCHIVE_FILTERS = new ArrayList<>(); + static { + ARCHIVE_FILTERS.add(SEVEN_ZIP_FILTER); + } private ArchiveUtil() { } + static List getArchiveFilters() { + return ARCHIVE_FILTERS; + } + + static boolean isArchive(Path dataSourcePath) { + String fileName = dataSourcePath.getFileName().toString(); + // check whether it's a zip archive file that can be extracted + return isAcceptedByFiler(new File(fileName), ARCHIVE_FILTERS); + } + + private static boolean isAcceptedByFiler(File file, List filters) { + for (FileFilter filter : filters) { + if (filter.accept(file)) { + return true; + } + } + return false; + } + /** * Enum of mime types which support archive extraction */ From 0fcea2306670bffda9e35b7bd569c50e9352e484 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 19 Oct 2017 16:58:26 -0400 Subject: [PATCH 23/30] Removed a log entry to log available disk space every minute --- Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index f6580a3cbc..08c619209f 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -192,7 +192,7 @@ public final class IngestMonitor { } logMemoryUsage(); - logDiskSpaceUsage(); + //logDiskSpaceUsage(); // this creates a log entry every minute if (!enoughDiskSpace()) { /* From 67cab4672fa99ea3cb1fe72ca30087e5efdd58bb Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 20 Oct 2017 13:29:20 -0400 Subject: [PATCH 24/30] Fix add/remove property change listener code for KeywordSearch UI classes --- .../autopsy/keywordsearch/GlobalListSettingsPanel.java | 2 ++ .../keywordsearch/KeywordSearchGlobalSettingsPanel.java | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/GlobalListSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/GlobalListSettingsPanel.java index cb7aac9018..e28cd75654 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/GlobalListSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/GlobalListSettingsPanel.java @@ -138,12 +138,14 @@ final class GlobalListSettingsPanel extends javax.swing.JPanel implements Option @Override public void addPropertyChangeListener(PropertyChangeListener l) { + super.addPropertyChangeListener(l); listsManagementPanel.addPropertyChangeListener(l); editListPanel.addPropertyChangeListener(l); } @Override public void removePropertyChangeListener(PropertyChangeListener l) { + super.removePropertyChangeListener(l); listsManagementPanel.removePropertyChangeListener(l); editListPanel.removePropertyChangeListener(l); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java index 1756bfc1ec..93fa053dd4 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,11 +28,12 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; */ final class KeywordSearchGlobalSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel { + private static final long serialVersionUID = 1L; private GlobalListSettingsPanel listsPanel; private KeywordSearchGlobalLanguageSettingsPanel languagesPanel; private KeywordSearchGlobalSearchSettingsPanel generalPanel; - public KeywordSearchGlobalSettingsPanel() { + KeywordSearchGlobalSettingsPanel() { initComponents(); customizeComponents(); } @@ -53,6 +54,7 @@ final class KeywordSearchGlobalSettingsPanel extends IngestModuleGlobalSettingsP @Override public void addPropertyChangeListener(PropertyChangeListener l) { + super.addPropertyChangeListener(l); listsPanel.addPropertyChangeListener(l); languagesPanel.addPropertyChangeListener(l); generalPanel.addPropertyChangeListener(l); @@ -60,6 +62,7 @@ final class KeywordSearchGlobalSettingsPanel extends IngestModuleGlobalSettingsP @Override public void removePropertyChangeListener(PropertyChangeListener l) { + super.removePropertyChangeListener(l); listsPanel.removePropertyChangeListener(l); languagesPanel.removePropertyChangeListener(l); generalPanel.removePropertyChangeListener(l); From 19a8249fddc15494d00d82d0a83d64de14c5fc41 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 23 Oct 2017 14:34:28 -0400 Subject: [PATCH 25/30] Extracting contents of archive to case output folder --- .../autoingest/AddArchiveTask.java | 53 +++++++++---------- .../ArchiveExtractorDSProcessor.java | 14 ++--- .../autoingest/ArchiveFilePanel.java | 12 ++--- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 4d744803c9..3a92d14452 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -22,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import org.apache.commons.io.FilenameUtils; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; @@ -35,43 +36,30 @@ public class AddArchiveTask implements Runnable { private final Logger logger = Logger.getLogger(AddArchiveTask.class.getName()); private final String deviceId; - private final String imagePath; + private final String archivePath; private final DataSourceProcessorProgressMonitor progressMonitor; private final DataSourceProcessorCallback callback; private boolean criticalErrorOccurred; - + private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; - /* - * The cancellation requested flag and SleuthKit add image process are - * guarded by a monitor (called a lock here to avoid confusion with the - * progress monitor) to synchronize cancelling the process (setting the flag - * and calling its stop method) and calling either its commit or revert - * method. The built-in monitor of the add image process can't be used for - * this because it is already used to synchronize its run (init part), - * commit, revert, and currentDirectory methods. - * - * TODO (AUT-2021): Merge SleuthkitJNI.AddImageProcess and AddImageTask - */ - private final Object tskAddImageProcessLock; - /** - * Constructs a runnable task that adds an image to the case database. + * Constructs a runnable task that adds an archive and data sources + * contained in the archive to the case database. * * @param deviceId An ASCII-printable identifier for the device associated * with the data source that is intended to be unique across multiple cases * (e.g., a UUID). - * @param imagePath Path to the image file. + * @param archivePath Path to the archive file. * @param progressMonitor Progress monitor to report progress during * processing. * @param callback Callback to call when processing is done. */ - AddArchiveTask(String deviceId, String imagePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + AddArchiveTask(String deviceId, String archivePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { this.deviceId = deviceId; - this.imagePath = imagePath; + this.archivePath = archivePath; this.callback = callback; this.progressMonitor = progressMonitor; - tskAddImageProcessLock = new Object(); } /** @@ -79,9 +67,9 @@ public class AddArchiveTask implements Runnable { */ @Override public void run() { - if (!ArchiveUtil.isArchive(Paths.get(imagePath))) { + if (!ArchiveUtil.isArchive(Paths.get(archivePath))) { List errorMessages = new ArrayList<>(); - errorMessages.add("Input data source is not a valid datasource: " + imagePath.toString()); + errorMessages.add("Input data source is not a valid datasource: " + archivePath.toString()); List newDataSources = new ArrayList<>(); DataSourceProcessorCallback.DataSourceProcessorResult result; result = DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; @@ -89,22 +77,31 @@ public class AddArchiveTask implements Runnable { } // extract the archive and pass the extracted folder as input - Path extractedDataSourcePath = Paths.get(""); + Path destinationFolder = Paths.get(""); try { Case currentCase = Case.getCurrentCase(); - //extractedDataSourcePath = extractDataSource(Paths.get(currentCase.getModuleDirectory()), imagePath); + + // get file name without extension + String dataSourceFileNameNoExt = FilenameUtils.removeExtension(archivePath); + + // create folder to extract archive to + destinationFolder = Paths.get(currentCase.getModuleDirectory(), dataSourceFileNameNoExt + "_" + TimeStampUtils.createTimeStamp()); + destinationFolder.toFile().mkdirs(); + + // extract contents of ZIP archive into destination folder + ArchiveUtil.unpackArchiveFile(archivePath, destinationFolder.toString()); } catch (Exception ex) { //throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException(NbBundle.getMessage(ArchiveExtractorDSProcessor.class, "ArchiveExtractorDataSourceProcessor.process.exception.text"), ex); } // do processing + return; } - - + /* - * Attempts to cancel adding the image to the case database. + * Attempts to cancel adding the archive to the case database. */ public void cancelTask() { - + } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java index 6fa8238dc3..2eb066a366 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java @@ -45,7 +45,7 @@ public class ArchiveExtractorDSProcessor implements DataSourceProcessor, AutoIng private final ArchiveFilePanel configPanel; private String deviceId; - private String imagePath; + private String archivePath; private boolean setDataSourceOptionsCalled; private AddArchiveTask addArchiveTask; @@ -126,9 +126,9 @@ public class ArchiveExtractorDSProcessor implements DataSourceProcessor, AutoIng if (!setDataSourceOptionsCalled) { configPanel.storeSettings(); deviceId = UUID.randomUUID().toString(); - imagePath = configPanel.getContentPaths(); + archivePath = configPanel.getContentPaths(); } - run(deviceId, imagePath, progressMonitor, callback); + run(deviceId, archivePath, progressMonitor, callback); } /** @@ -142,13 +142,13 @@ public class ArchiveExtractorDSProcessor implements DataSourceProcessor, AutoIng * associated with the data source that is * intended to be unique across multiple cases * (e.g., a UUID). - * @param imagePath Path to the image file. + * @param archivePath Path to the archive file. * @param progressMonitor Progress monitor for reporting progress * during processing. * @param callback Callback to call when processing is done. */ - public void run(String deviceId, String imagePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - addArchiveTask = new AddArchiveTask(deviceId, imagePath, progressMonitor, callback); + public void run(String deviceId, String archivePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + addArchiveTask = new AddArchiveTask(deviceId, archivePath, progressMonitor, callback); new Thread(addArchiveTask).start(); } @@ -169,7 +169,7 @@ public class ArchiveExtractorDSProcessor implements DataSourceProcessor, AutoIng @Override public void reset() { deviceId = null; - imagePath = null; + archivePath = null; configPanel.reset(); setDataSourceOptionsCalled = false; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java index e368a18c4f..e4b261ea3b 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java @@ -45,7 +45,7 @@ import org.sleuthkit.autopsy.coreutils.PathValidator; public class ArchiveFilePanel extends JPanel implements DocumentListener { private static final Logger logger = Logger.getLogger(ArchiveFilePanel.class.getName()); - private static final String PROP_LASTIMAGE_PATH = "LBL_LastImage_PATH"; //NON-NLS + private static final String PROP_LAST_ARCHIVE_PATH = "LBL_LastImage_PATH"; //NON-NLS private final JFileChooser fileChooser = new JFileChooser(); @@ -155,7 +155,7 @@ public class ArchiveFilePanel extends JPanel implements DocumentListener { private void browseButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_browseButtonActionPerformed String oldText = getContentPaths(); - // set the current directory of the FileChooser if the ImagePath Field is valid + // set the current directory of the FileChooser if the ArchivePath Field is valid File currentDir = new File(oldText); if (currentDir.exists()) { fileChooser.setCurrentDirectory(currentDir); @@ -179,7 +179,7 @@ public class ArchiveFilePanel extends JPanel implements DocumentListener { /** * Get the path of the user selected archive. * - * @return the image path + * @return the archive path */ public String getContentPaths() { return pathTextField.getText(); @@ -202,7 +202,7 @@ public class ArchiveFilePanel extends JPanel implements DocumentListener { /** * Should we enable the next button of the wizard? * - * @return true if a proper image has been selected, false otherwise + * @return true if a proper archive has been selected, false otherwise */ @NbBundle.Messages("DataSourceOnCDriveError.text=Warning: Path to multi-user data source is on \"C:\" drive") public boolean validatePanel() { @@ -227,12 +227,12 @@ public class ArchiveFilePanel extends JPanel implements DocumentListener { String archivePathName = getContentPaths(); if (null != archivePathName) { String archivePath = archivePathName.substring(0, archivePathName.lastIndexOf(File.separator) + 1); - ModuleSettings.setConfigSetting(contextName, PROP_LASTIMAGE_PATH, archivePath); + ModuleSettings.setConfigSetting(contextName, PROP_LAST_ARCHIVE_PATH, archivePath); } } public void readSettings() { - String lastArchivePath = ModuleSettings.getConfigSetting(contextName, PROP_LASTIMAGE_PATH); + String lastArchivePath = ModuleSettings.getConfigSetting(contextName, PROP_LAST_ARCHIVE_PATH); if (StringUtils.isNotBlank(lastArchivePath)) { setContentPath(lastArchivePath); } From 0f309e15b5b5dec0d62626d1fe6b777811e65885 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 23 Oct 2017 14:57:12 -0400 Subject: [PATCH 26/30] Fixes and improvements --- .../autoingest/AddArchiveTask.java | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 3a92d14452..2ad57bde9a 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -22,7 +22,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import org.apache.commons.io.FilenameUtils; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; @@ -41,7 +43,7 @@ public class AddArchiveTask implements Runnable { private final DataSourceProcessorCallback callback; private boolean criticalErrorOccurred; - private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; + private static final String ARCHIVE_EXTRACTOR_MODULE_OUTPUT_DIR = "Archive Extractor"; /** * Constructs a runnable task that adds an archive and data sources @@ -67,11 +69,13 @@ public class AddArchiveTask implements Runnable { */ @Override public void run() { + List errorMessages = new ArrayList<>(); + List newDataSources = new ArrayList<>(); + DataSourceProcessorCallback.DataSourceProcessorResult result; if (!ArchiveUtil.isArchive(Paths.get(archivePath))) { - List errorMessages = new ArrayList<>(); - errorMessages.add("Input data source is not a valid datasource: " + archivePath.toString()); - List newDataSources = new ArrayList<>(); - DataSourceProcessorCallback.DataSourceProcessorResult result; + criticalErrorOccurred = true; + logger.log(Level.SEVERE, String.format("Input data source is not a valid datasource: %s", archivePath)); //NON-NLS + errorMessages.add("Input data source is not a valid datasource: " + archivePath); result = DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; callback.done(result, errorMessages, newDataSources); } @@ -81,21 +85,32 @@ public class AddArchiveTask implements Runnable { try { Case currentCase = Case.getCurrentCase(); - // get file name without extension - String dataSourceFileNameNoExt = FilenameUtils.removeExtension(archivePath); + // get file name without full path or extension + String dataSourceFileNameNoExt = FilenameUtils.getBaseName(archivePath); // create folder to extract archive to - destinationFolder = Paths.get(currentCase.getModuleDirectory(), dataSourceFileNameNoExt + "_" + TimeStampUtils.createTimeStamp()); + destinationFolder = Paths.get(currentCase.getModuleDirectory(), ARCHIVE_EXTRACTOR_MODULE_OUTPUT_DIR, dataSourceFileNameNoExt + "_" + TimeStampUtils.createTimeStamp()); destinationFolder.toFile().mkdirs(); // extract contents of ZIP archive into destination folder ArchiveUtil.unpackArchiveFile(archivePath, destinationFolder.toString()); - } catch (Exception ex) { - //throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException(NbBundle.getMessage(ArchiveExtractorDSProcessor.class, "ArchiveExtractorDataSourceProcessor.process.exception.text"), ex); + + // do processing + + } catch (ArchiveUtil.ArchiveExtractionException ex) { + criticalErrorOccurred = true; + errorMessages.add(ex.getMessage()); + logger.log(Level.SEVERE, String.format("Critical error occurred while extracting archive %s", archivePath), ex); //NON-NLS + } finally { + if (criticalErrorOccurred) { + result = DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; + } else if (!errorMessages.isEmpty()) { + result = DataSourceProcessorCallback.DataSourceProcessorResult.NONCRITICAL_ERRORS; + } else { + result = DataSourceProcessorCallback.DataSourceProcessorResult.NO_ERRORS; + } + callback.done(result, errorMessages, newDataSources); } - - // do processing - return; } /* From 5f9c5facce8c7c399760f6123d9c9b38280eae49 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 23 Oct 2017 15:20:12 -0400 Subject: [PATCH 27/30] Commented out some functionality for now --- .../experimental/autoingest/AddArchiveTask.java | 10 +++++----- .../autoingest/ArchiveExtractorDSProcessor.java | 8 ++++---- .../experimental/autoingest/AutoIngestManager.java | 1 - 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 2ad57bde9a..54e8ec40f7 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import org.apache.commons.io.FilenameUtils; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; @@ -32,7 +31,8 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; /* - * A runnable that adds an archive data source to the case database. + * A runnable that adds an archive data source as well as data sources + * contained in the archive to the case database. */ public class AddArchiveTask implements Runnable { @@ -46,7 +46,7 @@ public class AddArchiveTask implements Runnable { private static final String ARCHIVE_EXTRACTOR_MODULE_OUTPUT_DIR = "Archive Extractor"; /** - * Constructs a runnable task that adds an archive and data sources + * Constructs a runnable task that adds an archive as well as data sources * contained in the archive to the case database. * * @param deviceId An ASCII-printable identifier for the device associated @@ -93,11 +93,11 @@ public class AddArchiveTask implements Runnable { destinationFolder.toFile().mkdirs(); // extract contents of ZIP archive into destination folder - ArchiveUtil.unpackArchiveFile(archivePath, destinationFolder.toString()); + //ArchiveUtil.unpackArchiveFile(archivePath, destinationFolder.toString()); // do processing - } catch (ArchiveUtil.ArchiveExtractionException ex) { + } catch (Exception ex) { criticalErrorOccurred = true; errorMessages.add(ex.getMessage()); logger.log(Level.SEVERE, String.format("Critical error occurred while extracting archive %s", archivePath), ex); //NON-NLS diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java index 2eb066a366..0103815e1a 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java @@ -35,10 +35,10 @@ import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; * add data source wizard. It also provides a run method overload to allow it to * be used independently of the wizard. */ -@ServiceProviders(value={ - @ServiceProvider(service=DataSourceProcessor.class), - @ServiceProvider(service=AutoIngestDataSourceProcessor.class)} -) +//@ServiceProviders(value={ +// @ServiceProvider(service=DataSourceProcessor.class), +// @ServiceProvider(service=AutoIngestDataSourceProcessor.class)} +//) public class ArchiveExtractorDSProcessor implements DataSourceProcessor, AutoIngestDataSourceProcessor { private final static String DATA_SOURCE_TYPE = NbBundle.getMessage(ArchiveExtractorDSProcessor.class, "ArchiveExtractorDataSourceProcessor.dsType.text"); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 7ffb67ae05..8d7df1b6c0 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -1449,7 +1449,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ private final class JobProcessingTask implements Runnable { - private static final String AUTO_INGEST_MODULE_OUTPUT_DIR = "AutoIngest"; private final Object ingestLock; private final Object pauseLock; @GuardedBy("pauseLock") From 0451208df814b432dda3fae51a1028b7a41cb209 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 23 Oct 2017 16:28:52 -0400 Subject: [PATCH 28/30] Code review comments --- Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java | 2 -- .../autopsy/experimental/autoingest/AddArchiveTask.java | 2 +- .../autoingest/ArchiveExtractorDSProcessor.java | 6 ++++-- .../autopsy/experimental/autoingest/ArchiveFilePanel.java | 2 +- .../autopsy/experimental/autoingest/AutoIngestManager.java | 2 +- .../autopsy/experimental/autoingest/Bundle.properties | 2 -- ...ourceProcessors.java => DataSourceProcessorUtility.java} | 5 ++++- 7 files changed, 11 insertions(+), 10 deletions(-) rename Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/{IdentifyDataSourceProcessors.java => DataSourceProcessorUtility.java} (96%) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index 08c619209f..8aeda52597 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -192,8 +192,6 @@ public final class IngestMonitor { } logMemoryUsage(); - //logDiskSpaceUsage(); // this creates a log entry every minute - if (!enoughDiskSpace()) { /* * Shut down ingest by cancelling all ingest jobs. diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 54e8ec40f7..3bc76e61fd 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -34,7 +34,7 @@ import org.sleuthkit.datamodel.Content; * A runnable that adds an archive data source as well as data sources * contained in the archive to the case database. */ -public class AddArchiveTask implements Runnable { +class AddArchiveTask implements Runnable { private final Logger logger = Logger.getLogger(AddArchiveTask.class.getName()); private final String deviceId; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java index 0103815e1a..a649b0f37a 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveExtractorDSProcessor.java @@ -39,9 +39,11 @@ import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; // @ServiceProvider(service=DataSourceProcessor.class), // @ServiceProvider(service=AutoIngestDataSourceProcessor.class)} //) +@NbBundle.Messages({ + "ArchiveDSP.dsType.text=Archive file"}) public class ArchiveExtractorDSProcessor implements DataSourceProcessor, AutoIngestDataSourceProcessor { - - private final static String DATA_SOURCE_TYPE = NbBundle.getMessage(ArchiveExtractorDSProcessor.class, "ArchiveExtractorDataSourceProcessor.dsType.text"); + + private final static String DATA_SOURCE_TYPE = Bundle.ArchiveDSP_dsType_text(); private final ArchiveFilePanel configPanel; private String deviceId; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java index e4b261ea3b..e5285d9e75 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/ArchiveFilePanel.java @@ -42,7 +42,7 @@ import org.sleuthkit.autopsy.coreutils.PathValidator; * "zip", "rar", "arj", "7z", "7zip", "gzip, etc). Allows the user to select a * file. */ -public class ArchiveFilePanel extends JPanel implements DocumentListener { +class ArchiveFilePanel extends JPanel implements DocumentListener { private static final Logger logger = Logger.getLogger(ArchiveFilePanel.class.getName()); private static final String PROP_LAST_ARCHIVE_PATH = "LBL_LastImage_PATH"; //NON-NLS diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 8d7df1b6c0..2f2f22fb0d 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2316,7 +2316,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang Map validDataSourceProcessorsMap; try { // lookup all AutomatedIngestDataSourceProcessors and poll which ones are able to process the current data source - validDataSourceProcessorsMap = IdentifyDataSourceProcessors.getDataSourceProcessor(dataSource.getPath()); + validDataSourceProcessorsMap = DataSourceProcessorUtility.getDataSourceProcessor(dataSource.getPath()); } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { SYS_LOGGER.log(Level.SEVERE, "Exception while determining best data source processor for {0}", dataSource.getPath()); // rethrow the exception. It will get caught & handled upstream and will result in AIM auto-pause. diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties index ec32087dbf..edac418672 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties @@ -247,8 +247,6 @@ AutoIngestDashboard.prioritizeJobButton.toolTipText=Move the selected job to the AutoIngestDashboard.prioritizeJobButton.text=Prioritize &Job AutoIngestDashboard.prioritizeCaseButton.toolTipText=Move all images associated with a case to top of Pending queue. AutoIngestDashboard.prioritizeCaseButton.text=Prioritize &Case -ArchiveExtractorDataSourceProcessor.dsType.text=Archive file -ArchiveExtractorDataSourceProcessor.process.exception.text=Exception while trying to extract archive ArchiveFilePanel.pathLabel.text=Browse for an archive file: ArchiveFilePanel.browseButton.text=Browse ArchiveFilePanel.pathTextField.text= diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSourceProcessorUtility.java similarity index 96% rename from Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java rename to Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSourceProcessorUtility.java index b566d99aac..6f88d70a28 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/IdentifyDataSourceProcessors.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DataSourceProcessorUtility.java @@ -29,7 +29,10 @@ import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor. /** * A utility class to find Data Source Processors */ -class IdentifyDataSourceProcessors { +class DataSourceProcessorUtility { + + private DataSourceProcessorUtility() { + } /** * A utility method to find all Data Source Processors (DSP) that are able From 94366cc8eab74954971c0f4e697934e4b73ec020 Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 24 Oct 2017 10:48:11 -0400 Subject: [PATCH 29/30] 857: Reconcile Unique Accounts with Credit Card Review --- .../autopsy/datamodel/accounts/Accounts.java | 9 +- .../autopsy/keywordsearch/RegexQuery.java | 204 ++++++++++-------- .../keywordsearch/TermsComponentQuery.java | 204 ++++++++++-------- 3 files changed, 245 insertions(+), 172 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java index b27ffd9d38..e9ab484b3d 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java @@ -1429,7 +1429,14 @@ final public class Accounts implements AutopsyVisitableItem { artifacts.forEach(artifact -> { try { AccountInstance accountInstance = skCase.getCommunicationsManager().getAccountInstance(artifact); - accountInstance.setReviewStatus(newStatus); + + if (BlackboardArtifact.ReviewStatus.APPROVED == newStatus) { + accountInstance.approveAccount(); + } + else { + accountInstance.rejectAccount(); + } + } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error changing artifact review status.", ex); //NON-NLS } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index a456e6c029..1d4f32f18b 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -412,97 +412,30 @@ final class RegexQuery implements KeywordSearchQuery { return null; } + /* + * Credit Card number hits are handled differently + */ + if (originalKeyword.getArtifactAttributeType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { + createCCNAccount(content, foundKeyword, hit, snippet, listName); + return null; + } /* * Create either a "plain vanilla" keyword hit artifact with keyword and - * regex attributes, or a credit card account artifact with attributes - * parsed from from the snippet for the hit and looked up based on the - * parsed bank identifcation number. + * regex attributes */ BlackboardArtifact newArtifact; Collection attributes = new ArrayList<>(); - if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString())); - try { - newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS - return null; - } - } else { - /* - * Parse the credit card account attributes from the snippet for the - * hit. - */ - Map parsedTrackAttributeMap = new HashMap<>(); - Matcher matcher = TermsComponentQuery.CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrack1Data(parsedTrackAttributeMap, matcher); - } - matcher = CREDIT_CARD_TRACK2_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrack2Data(parsedTrackAttributeMap, matcher); - } - final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)); - if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) { - if (hit.isArtifactHit()) { - LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS - } else { - LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getContentID())); //NON-NLS - } - return null; - } - attributes.addAll(parsedTrackAttributeMap.values()); - - /* - * Look up the bank name, scheme, etc. attributes for the bank - * indentification number (BIN). - */ - final int bin = Integer.parseInt(ccnAttribute.getValueString().substring(0, 8)); - CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin); - if (binInfo != null) { - binInfo.getScheme().ifPresent(scheme - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme))); - binInfo.getCardType().ifPresent(cardType - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType))); - binInfo.getBrand().ifPresent(brand - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand))); - binInfo.getBankName().ifPresent(bankName - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName))); - binInfo.getBankPhoneNumber().ifPresent(phoneNumber - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber))); - binInfo.getBankURL().ifPresent(url - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url))); - binInfo.getCountry().ifPresent(country - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country))); - binInfo.getBankCity().ifPresent(city - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city))); - } - - /* - * If the hit is from unused or unallocated space, record the Solr - * document id to support showing just the chunk that contained the - * hit. - */ - if (content instanceof AbstractFile) { - AbstractFile file = (AbstractFile) content; - if (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS - || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) { - attributes.add(new BlackboardAttribute(KEYWORD_SEARCH_DOCUMENT_ID, MODULE_NAME, hit.getSolrDocumentId())); - } - } - - /* - * Create an account instance. - */ - try { - AccountInstance ccAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountInstance(Account.Type.CREDIT_CARD, ccnAttribute.getValueString() , MODULE_NAME, content); - newArtifact = Case.getCurrentCase().getSleuthkitCase().getBlackboardArtifact(ccAccountInstance.getArtifactId()); - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error adding artifact for account to blackboard", ex); //NON-NLS - return null; - } + + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString())); + + try { + newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS + return null; } + if (StringUtils.isNotBlank(listName)) { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); @@ -526,6 +459,107 @@ final class RegexQuery implements KeywordSearchQuery { } } + private void createCCNAccount(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + + final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); + + if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { + LOGGER.log(Level.SEVERE, "Keyword hit is not a credit card number"); //NON-NLS + return; + } + /* + * Create a credit card account with attributes + * parsed from the snippet for the hit and looked up based on the + * parsed bank identifcation number. + */ + Collection attributes = new ArrayList<>(); + + Map parsedTrackAttributeMap = new HashMap<>(); + Matcher matcher = TermsComponentQuery.CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrack1Data(parsedTrackAttributeMap, matcher); + } + matcher = CREDIT_CARD_TRACK2_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrack2Data(parsedTrackAttributeMap, matcher); + } + final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)); + if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) { + if (hit.isArtifactHit()) { + LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS + } else { + LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getContentID())); //NON-NLS + } + return; + } + attributes.addAll(parsedTrackAttributeMap.values()); + + /* + * Look up the bank name, scheme, etc. attributes for the bank + * indentification number (BIN). + */ + final int bin = Integer.parseInt(ccnAttribute.getValueString().substring(0, 8)); + CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin); + if (binInfo != null) { + binInfo.getScheme().ifPresent(scheme + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme))); + binInfo.getCardType().ifPresent(cardType + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType))); + binInfo.getBrand().ifPresent(brand + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand))); + binInfo.getBankName().ifPresent(bankName + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName))); + binInfo.getBankPhoneNumber().ifPresent(phoneNumber + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber))); + binInfo.getBankURL().ifPresent(url + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url))); + binInfo.getCountry().ifPresent(country + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country))); + binInfo.getBankCity().ifPresent(city + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city))); + } + + /* + * If the hit is from unused or unallocated space, record the Solr + * document id to support showing just the chunk that contained the + * hit. + */ + if (content instanceof AbstractFile) { + AbstractFile file = (AbstractFile) content; + if (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS + || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) { + attributes.add(new BlackboardAttribute(KEYWORD_SEARCH_DOCUMENT_ID, MODULE_NAME, hit.getSolrDocumentId())); + } + } + + if (StringUtils.isNotBlank(listName)) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); + } + if (snippet != null) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); + } + + hit.getArtifactID().ifPresent(artifactID + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) + ); + + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal())); + + + /* + * Create an account instance. + */ + try { + AccountInstance ccAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountInstance(Account.Type.CREDIT_CARD, ccnAttribute.getValueString() , MODULE_NAME, content); + + ccAccountInstance.addAttributes(attributes); + + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error creating CCN account instance", ex); //NON-NLS + + } + + } /** * Parses the track 2 data from the snippet for a credit card account number * hit and turns them into artifact attributes. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java index c8ae73dc78..fb1bc3c3e4 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java @@ -329,99 +329,33 @@ final class TermsComponentQuery implements KeywordSearchQuery { @Override public BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) { + /* - * Create either a "plain vanilla" keyword hit artifact with keyword and - * regex attributes, or a credit card account artifact with attributes - * parsed from from the snippet for the hit and looked up based on the - * parsed bank identifcation number. + * CCN hits are handled specially + */ + if (originalKeyword.getArtifactAttributeType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { + createCCNAccount(content, hit, snippet, listName); + return null; + } + + /* + * Create a "plain vanilla" keyword hit artifact with keyword and + * regex attributes, */ BlackboardArtifact newArtifact; Collection attributes = new ArrayList<>(); - if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, originalKeyword.getSearchTerm())); + + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, originalKeyword.getSearchTerm())); - try { - newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); + try { + newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS - return null; - } - } else { - /* - * Parse the credit card account attributes from the snippet for the - * hit. - */ - Map parsedTrackAttributeMap = new HashMap<>(); - Matcher matcher = CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrack1Data(parsedTrackAttributeMap, matcher); - } - matcher = CREDIT_CARD_TRACK2_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrack2Data(parsedTrackAttributeMap, matcher); - } - final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)); - if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) { - if (hit.isArtifactHit()) { - LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", searchTerm, hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS - } else { - LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", searchTerm, hit.getSnippet(), hit.getContentID())); //NON-NLS - } - return null; - } - attributes.addAll(parsedTrackAttributeMap.values()); - - /* - * Look up the bank name, scheme, etc. attributes for the bank - * indentification number (BIN). - */ - final int bin = Integer.parseInt(ccnAttribute.getValueString().substring(0, 8)); - CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin); - if (binInfo != null) { - binInfo.getScheme().ifPresent(scheme - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme))); - binInfo.getCardType().ifPresent(cardType - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType))); - binInfo.getBrand().ifPresent(brand - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand))); - binInfo.getBankName().ifPresent(bankName - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName))); - binInfo.getBankPhoneNumber().ifPresent(phoneNumber - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber))); - binInfo.getBankURL().ifPresent(url - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url))); - binInfo.getCountry().ifPresent(country - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country))); - binInfo.getBankCity().ifPresent(city - -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city))); - } - - /* - * If the hit is from unused or unallocated space, record the Solr - * document id to support showing just the chunk that contained the - * hit. - */ - if (content instanceof AbstractFile) { - AbstractFile file = (AbstractFile) content; - if (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS - || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) { - attributes.add(new BlackboardAttribute(KEYWORD_SEARCH_DOCUMENT_ID, MODULE_NAME, hit.getSolrDocumentId())); - } - } - - /* - * Create an account artifact. - */ - try { - AccountInstance ccAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountInstance(Account.Type.CREDIT_CARD, ccnAttribute.getValueString() , MODULE_NAME, content); - newArtifact = Case.getCurrentCase().getSleuthkitCase().getBlackboardArtifact(ccAccountInstance.getArtifactId()); - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error adding artifact for account to blackboard", ex); //NON-NLS - return null; - } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error adding artifact for keyword hit to blackboard", ex); //NON-NLS + return null; } + if (StringUtils.isNotBlank(listName)) { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); @@ -446,6 +380,104 @@ final class TermsComponentQuery implements KeywordSearchQuery { } } + private void createCCNAccount(Content content, KeywordHit hit, String snippet, String listName) { + + if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { + LOGGER.log(Level.SEVERE, "Keyword hit is not a credit card number"); //NON-NLS + return; + } + + /* + * Create a credit card account with attributes + * parsed from from the snippet for the hit and looked up based on the + * parsed bank identifcation number. + */ + Collection attributes = new ArrayList<>(); + + Map parsedTrackAttributeMap = new HashMap<>(); + Matcher matcher = CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrack1Data(parsedTrackAttributeMap, matcher); + } + matcher = CREDIT_CARD_TRACK2_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrack2Data(parsedTrackAttributeMap, matcher); + } + final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)); + if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) { + if (hit.isArtifactHit()) { + LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", searchTerm, hit.getSnippet(), hit.getArtifactID().get())); //NON-NLS + } else { + LOGGER.log(Level.SEVERE, String.format("Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", searchTerm, hit.getSnippet(), hit.getContentID())); //NON-NLS + } + return; + } + attributes.addAll(parsedTrackAttributeMap.values()); + + /* + * Look up the bank name, scheme, etc. attributes for the bank + * indentification number (BIN). + */ + final int bin = Integer.parseInt(ccnAttribute.getValueString().substring(0, 8)); + CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin); + if (binInfo != null) { + binInfo.getScheme().ifPresent(scheme + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme))); + binInfo.getCardType().ifPresent(cardType + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType))); + binInfo.getBrand().ifPresent(brand + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand))); + binInfo.getBankName().ifPresent(bankName + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName))); + binInfo.getBankPhoneNumber().ifPresent(phoneNumber + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber))); + binInfo.getBankURL().ifPresent(url + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url))); + binInfo.getCountry().ifPresent(country + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country))); + binInfo.getBankCity().ifPresent(city + -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city))); + } + + /* + * If the hit is from unused or unallocated space, record the Solr + * document id to support showing just the chunk that contained the + * hit. + */ + if (content instanceof AbstractFile) { + AbstractFile file = (AbstractFile) content; + if (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS + || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) { + attributes.add(new BlackboardAttribute(KEYWORD_SEARCH_DOCUMENT_ID, MODULE_NAME, hit.getSolrDocumentId())); + } + } + + if (StringUtils.isNotBlank(listName)) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); + } + if (snippet != null) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); + } + + hit.getArtifactID().ifPresent( + artifactID -> attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID)) + ); + + // TermsComponentQuery is now being used exclusively for substring searches. + attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal())); + + /* + * Create an account. + */ + try { + AccountInstance ccAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountInstance(Account.Type.CREDIT_CARD, ccnAttribute.getValueString() , MODULE_NAME, content); + ccAccountInstance.addAttributes(attributes); + //newArtifact = Case.getCurrentCase().getSleuthkitCase().getBlackboardArtifact(ccAccountInstance.getArtifactId()); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error creating CCN account instance", ex); //NON-NLS + } + + } /** * Parses the track 2 data from the snippet for a credit card account number * hit and turns them into artifact attributes. From d829203dc70457c5476ac4e9fefbf65dfaca921a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 24 Oct 2017 13:36:52 -0400 Subject: [PATCH 30/30] Fix comment in RegexQuery class. --- .../src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index 1d4f32f18b..4dea228f22 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -419,8 +419,9 @@ final class RegexQuery implements KeywordSearchQuery { createCCNAccount(content, foundKeyword, hit, snippet, listName); return null; } + /* - * Create either a "plain vanilla" keyword hit artifact with keyword and + * Create a "plain vanilla" keyword hit artifact with keyword and * regex attributes */ BlackboardArtifact newArtifact;