From cdfe03290fcfcedb7e4e35fc912929ffb423377c Mon Sep 17 00:00:00 2001 From: adam-m Date: Wed, 12 Jun 2013 22:38:51 -0400 Subject: [PATCH 01/12] fix race condition likely causing null ptr exception when registering a listener. Fixes #218 --- .../org/sleuthkit/autopsy/casemodule/ImageFilePanel.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java index f7bc809a8c..542cba60d6 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java @@ -31,9 +31,9 @@ import javax.swing.event.DocumentListener; * ImageTypePanel for adding an image file such as .img, .E0x, .00x, etc. */ public class ImageFilePanel extends ContentTypePanel implements DocumentListener { - private static ImageFilePanel instance; - private PropertyChangeSupport pcs = new PropertyChangeSupport(this); - private JFileChooser fc = new JFileChooser(); + private static volatile ImageFilePanel instance = null; + private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); + private final JFileChooser fc = new JFileChooser(); /** * Creates new form ImageFilePanel @@ -53,7 +53,7 @@ public class ImageFilePanel extends ContentTypePanel implements DocumentListener /** * Returns the default instance of a ImageFilePanel. */ - public static ImageFilePanel getDefault() { + public static synchronized ImageFilePanel getDefault() { if (instance == null) { instance = new ImageFilePanel(); } From 9f1eaf3327c1c90987672253699eea816e67512d Mon Sep 17 00:00:00 2001 From: adam-m Date: Thu, 13 Jun 2013 13:07:02 -0400 Subject: [PATCH 02/12] fix in add source dialogs: initialize proeprty change support after object has been fully initialized --- .../org/sleuthkit/autopsy/casemodule/ImageFilePanel.java | 8 +++++++- .../org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java | 8 +++++++- .../sleuthkit/autopsy/casemodule/LocalFilesPanel.java | 9 +++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java index f7bc809a8c..1e9d3568f1 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java @@ -32,7 +32,7 @@ import javax.swing.event.DocumentListener; */ public class ImageFilePanel extends ContentTypePanel implements DocumentListener { private static ImageFilePanel instance; - private PropertyChangeSupport pcs = new PropertyChangeSupport(this); + private PropertyChangeSupport pcs; private JFileChooser fc = new JFileChooser(); /** @@ -56,9 +56,15 @@ public class ImageFilePanel extends ContentTypePanel implements DocumentListener public static ImageFilePanel getDefault() { if (instance == null) { instance = new ImageFilePanel(); + instance.init(); } return instance; } + + //post - constructor initialization + private void init() { + pcs = new PropertyChangeSupport(this); + } /** * This method is called from within the constructor to initialize the form. diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java index 1ac9e5ed5c..2a214f0b20 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java @@ -45,7 +45,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; */ public class LocalDiskPanel extends ContentTypePanel { private static LocalDiskPanel instance; - private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); + private PropertyChangeSupport pcs; private List disks = new ArrayList(); private LocalDiskModel model; private boolean enableNext = false; @@ -64,10 +64,16 @@ public class LocalDiskPanel extends ContentTypePanel { public static LocalDiskPanel getDefault() { if (instance == null) { instance = new LocalDiskPanel(); + instance.init(); } return instance; } + //post - constructor initialization + private void init() { + pcs = new PropertyChangeSupport(this); + } + private void customInit() { model = new LocalDiskModel(); diskComboBox.setModel(model); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java index d15f103acd..ebb4cec842 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java @@ -30,7 +30,7 @@ import javax.swing.JFileChooser; */ public class LocalFilesPanel extends ContentTypePanel { - private PropertyChangeSupport pcs = new PropertyChangeSupport(this); + private PropertyChangeSupport pcs; private Set currentFiles = new TreeSet(); //keep currents in a set to disallow duplicates per add private boolean enableNext = false; private static LocalFilesPanel instance; @@ -47,6 +47,7 @@ public class LocalFilesPanel extends ContentTypePanel { static synchronized LocalFilesPanel getDefault() { if (instance == null) { instance = new LocalFilesPanel(); + instance.init(); } return instance; } @@ -55,7 +56,11 @@ public class LocalFilesPanel extends ContentTypePanel { localFileChooser.setMultiSelectionEnabled(true); selectedPaths.setText(""); - + } + + //post - constructor initialization + private void init() { + pcs = new PropertyChangeSupport(this); } @Override From 3dec1b1e16e093d7b29c3de73d950a9a29ffc4b0 Mon Sep 17 00:00:00 2001 From: adam-m Date: Thu, 13 Jun 2013 13:35:25 -0400 Subject: [PATCH 03/12] better way to ensure pcs has been initialized --- .../org/sleuthkit/autopsy/casemodule/ImageFilePanel.java | 8 ++------ .../org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java | 7 +------ .../org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java | 8 +------- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java index 3a294232af..bcae84d76d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java @@ -32,7 +32,7 @@ import javax.swing.event.DocumentListener; */ public class ImageFilePanel extends ContentTypePanel implements DocumentListener { private static ImageFilePanel instance; - private PropertyChangeSupport pcs; + private final PropertyChangeSupport pcs = new PropertyChangeSupport(ImageFilePanel.class); private JFileChooser fc = new JFileChooser(); /** @@ -56,15 +56,11 @@ public class ImageFilePanel extends ContentTypePanel implements DocumentListener public static synchronized ImageFilePanel getDefault() { if (instance == null) { instance = new ImageFilePanel(); - instance.init(); } return instance; } - //post - constructor initialization - private void init() { - pcs = new PropertyChangeSupport(this); - } + /** * This method is called from within the constructor to initialize the form. diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java index 22466d2b27..4ffd0882bd 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java @@ -45,7 +45,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; */ public class LocalDiskPanel extends ContentTypePanel { private static LocalDiskPanel instance; - private PropertyChangeSupport pcs; + private final PropertyChangeSupport pcs = new PropertyChangeSupport(LocalDiskPanel.class); private List disks = new ArrayList(); private LocalDiskModel model; private boolean enableNext = false; @@ -64,15 +64,10 @@ public class LocalDiskPanel extends ContentTypePanel { public static synchronized LocalDiskPanel getDefault() { if (instance == null) { instance = new LocalDiskPanel(); - instance.init(); } return instance; } - //post - constructor initialization - private void init() { - pcs = new PropertyChangeSupport(this); - } private void customInit() { model = new LocalDiskModel(); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java index ebb4cec842..978c3bf50e 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java @@ -30,7 +30,7 @@ import javax.swing.JFileChooser; */ public class LocalFilesPanel extends ContentTypePanel { - private PropertyChangeSupport pcs; + private final PropertyChangeSupport pcs = new PropertyChangeSupport(LocalFilesPanel.class); private Set currentFiles = new TreeSet(); //keep currents in a set to disallow duplicates per add private boolean enableNext = false; private static LocalFilesPanel instance; @@ -47,7 +47,6 @@ public class LocalFilesPanel extends ContentTypePanel { static synchronized LocalFilesPanel getDefault() { if (instance == null) { instance = new LocalFilesPanel(); - instance.init(); } return instance; } @@ -58,11 +57,6 @@ public class LocalFilesPanel extends ContentTypePanel { } - //post - constructor initialization - private void init() { - pcs = new PropertyChangeSupport(this); - } - @Override public String getContentPaths() { //TODO consider interface change to return list of paths instead From 0b4c2ab72406128ca2c644c9046a9a126c9dfff6 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 13 Jun 2013 14:16:03 -0400 Subject: [PATCH 04/12] fix pcs initialization issues for good (linux) by lazy instantiating pcs before it's used --- .../autopsy/casemodule/ContentTypePanel.java | 4 ---- .../autopsy/casemodule/ImageFilePanel.java | 22 +++++++++++++++---- .../autopsy/casemodule/LocalDiskPanel.java | 12 ++++++++-- .../autopsy/casemodule/LocalFilesPanel.java | 15 ++++++++++--- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ContentTypePanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/ContentTypePanel.java index 0a8eb2c303..07664d9401 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ContentTypePanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ContentTypePanel.java @@ -68,9 +68,5 @@ abstract class ContentTypePanel extends JPanel { */ abstract public void select(); - @Override - abstract public void addPropertyChangeListener(PropertyChangeListener pcl); - @Override - abstract public void removePropertyChangeListener(PropertyChangeListener pcl); } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java index bcae84d76d..8c57567668 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java @@ -31,8 +31,8 @@ import javax.swing.event.DocumentListener; * ImageTypePanel for adding an image file such as .img, .E0x, .00x, etc. */ public class ImageFilePanel extends ContentTypePanel implements DocumentListener { - private static ImageFilePanel instance; - private final PropertyChangeSupport pcs = new PropertyChangeSupport(ImageFilePanel.class); + private static ImageFilePanel instance = null; + private PropertyChangeSupport pcs = null; private JFileChooser fc = new JFileChooser(); /** @@ -47,7 +47,6 @@ public class ImageFilePanel extends ContentTypePanel implements DocumentListener fc.addChoosableFileFilter(AddImageVisualPanel1.rawFilter); fc.addChoosableFileFilter(AddImageVisualPanel1.encaseFilter); fc.setFileFilter(AddImageVisualPanel1.allFilter); - pathTextField.getDocument().addDocumentListener(this); } /** @@ -56,9 +55,16 @@ public class ImageFilePanel extends ContentTypePanel implements DocumentListener public static synchronized ImageFilePanel getDefault() { if (instance == null) { instance = new ImageFilePanel(); + instance.postInit(); } return instance; } + + //post-constructor initialization to properly initialize listener support + //without leaking references of uninitialized objects + private void postInit() { + pathTextField.getDocument().addDocumentListener(this); + } @@ -219,12 +225,20 @@ public class ImageFilePanel extends ContentTypePanel implements DocumentListener } @Override - public void addPropertyChangeListener(PropertyChangeListener pcl) { + public synchronized void addPropertyChangeListener(PropertyChangeListener pcl) { + super.addPropertyChangeListener(pcl); + + if (pcs == null) { + pcs = new PropertyChangeSupport(this); + } + pcs.addPropertyChangeListener(pcl); } @Override public void removePropertyChangeListener(PropertyChangeListener pcl) { + super.removePropertyChangeListener(pcl); + pcs.removePropertyChangeListener(pcl); } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java index 4ffd0882bd..9c9c3f5fe6 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java @@ -45,7 +45,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; */ public class LocalDiskPanel extends ContentTypePanel { private static LocalDiskPanel instance; - private final PropertyChangeSupport pcs = new PropertyChangeSupport(LocalDiskPanel.class); + private PropertyChangeSupport pcs = null; private List disks = new ArrayList(); private LocalDiskModel model; private boolean enableNext = false; @@ -190,12 +190,20 @@ public class LocalDiskPanel extends ContentTypePanel { } @Override - public void addPropertyChangeListener(PropertyChangeListener pcl) { + public synchronized void addPropertyChangeListener(PropertyChangeListener pcl) { + super.addPropertyChangeListener(pcl); + + if (pcs == null) { + pcs = new PropertyChangeSupport(this); + } + pcs.addPropertyChangeListener(pcl); } @Override public void removePropertyChangeListener(PropertyChangeListener pcl) { + super.removePropertyChangeListener(pcl); + pcs.removePropertyChangeListener(pcl); } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java index 978c3bf50e..2357151d2b 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java @@ -30,7 +30,7 @@ import javax.swing.JFileChooser; */ public class LocalFilesPanel extends ContentTypePanel { - private final PropertyChangeSupport pcs = new PropertyChangeSupport(LocalFilesPanel.class); + private PropertyChangeSupport pcs = null; private Set currentFiles = new TreeSet(); //keep currents in a set to disallow duplicates per add private boolean enableNext = false; private static LocalFilesPanel instance; @@ -101,16 +101,25 @@ public class LocalFilesPanel extends ContentTypePanel { pcs.firePropertyChange(AddImageVisualPanel1.EVENT.UPDATE_UI.toString(), false, true); } - @Override - public void addPropertyChangeListener(PropertyChangeListener pcl) { + @Override + public synchronized void addPropertyChangeListener(PropertyChangeListener pcl) { + super.addPropertyChangeListener(pcl); + + if (pcs == null) { + pcs = new PropertyChangeSupport(this); + } + pcs.addPropertyChangeListener(pcl); } @Override public void removePropertyChangeListener(PropertyChangeListener pcl) { + super.removePropertyChangeListener(pcl); + pcs.removePropertyChangeListener(pcl); } + @Override public String toString() { return "Logical Files"; From 514d6b182d4d7bdadb1af683aa044884232b273a Mon Sep 17 00:00:00 2001 From: adam-m Date: Mon, 17 Jun 2013 10:27:52 -0400 Subject: [PATCH 05/12] update timeline comment re: memory issues --- .../org/sleuthkit/autopsy/timeline/Timeline.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java index 1c2d29400e..76221845f0 100644 --- a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java +++ b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java @@ -106,6 +106,14 @@ import org.sleuthkit.datamodel.TskCoreException; @ActionReferences(value = { @ActionReference(path = "Menu/Tools", position = 100)}) @NbBundle.Messages(value = "CTL_TimelineView=Generate Timeline") +/** + * The Timeline Action entry point. Collects data and pushes data to javafx widgets + * + * 2 known memory issues: + * - makeBodyFile() - stores all AbstractFile objects. It should either store IDs only, or iterate over result set (need API addition) + * - charts are also storing all AbstractFile objects. It should only keep datamodel objects in memory that pertain to current selection. + * And it should only be using IDs for the high level zoomed-out chart. + */ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, PropertyChangeListener { private static final Logger logger = Logger.getLogger(Timeline.class.getName()); @@ -927,6 +935,11 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, + "AND name != '..'"; List files = null; try { + //TODO this causes one of the Timelines memory limitations + //need another method findAllFileIdsWhere() not to store all file objects in memory + //then create the AbstractFile object as needed later + //OR pass in object reader to optional findAllFilesWhere(query, reader) + //to collect 1 object at a time (much like cursor, but hiding SQL layer) files = skCase.findAllFilesWhere(filesAndDirs); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error querying image files to make a body file: " + bodyFilePath, ex); From 5d47d58d8a3f323581eec3405eefe60acea1820b Mon Sep 17 00:00:00 2001 From: adam-m Date: Mon, 17 Jun 2013 10:37:25 -0400 Subject: [PATCH 06/12] update date on quick start guide --- docs/QuickStartGuide/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QuickStartGuide/index.html b/docs/QuickStartGuide/index.html index 8a6d9f9f03..46b762a4bc 100644 --- a/docs/QuickStartGuide/index.html +++ b/docs/QuickStartGuide/index.html @@ -11,7 +11,7 @@

Autopsy 3 Quick Start Guide

-

June 2012

+

June 2013

www.sleuthkit.org/autopsy/

From 4a242017e5e89eff128078de378aafdc6f3921c1 Mon Sep 17 00:00:00 2001 From: adam-m Date: Mon, 17 Jun 2013 15:19:34 -0400 Subject: [PATCH 07/12] reduce heap memory footprint in Timeline --- NEWS.txt | 1 + .../sleuthkit/autopsy/timeline/Timeline.java | 121 +++++++++--------- 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index 8d27e46e62..2ad093db0b 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -14,6 +14,7 @@ Bugfixes: - fixed result viewer for "File Search by MD5 Hash" - fix Solr, Timeline and RecentActivity issues with java 7.0.21 - Views->Recent Files showing inconsistent results when clicked many times +- reduced memory usage in Timeline ---------------- VERSION 3.0.5 -------------- diff --git a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java index 76221845f0..40e1968599 100644 --- a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java +++ b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java @@ -80,6 +80,7 @@ import org.openide.modules.ModuleInstall; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; import org.openide.nodes.Node; +import org.openide.util.Exceptions; import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; @@ -102,17 +103,14 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @ActionID(category = "Tools", id = "org.sleuthkit.autopsy.timeline.Timeline") -@ActionRegistration(displayName = "#CTL_MakeTimeline", lazy=false) +@ActionRegistration(displayName = "#CTL_MakeTimeline", lazy = false) @ActionReferences(value = { @ActionReference(path = "Menu/Tools", position = 100)}) @NbBundle.Messages(value = "CTL_TimelineView=Generate Timeline") /** - * The Timeline Action entry point. Collects data and pushes data to javafx widgets - * - * 2 known memory issues: - * - makeBodyFile() - stores all AbstractFile objects. It should either store IDs only, or iterate over result set (need API addition) - * - charts are also storing all AbstractFile objects. It should only keep datamodel objects in memory that pertain to current selection. - * And it should only be using IDs for the high level zoomed-out chart. + * The Timeline Action entry point. Collects data and pushes data to javafx + * widgets + * */ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, PropertyChangeListener { @@ -124,14 +122,14 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, private HBox fxHBoxCharts; //Holds the navigation buttons in horiztonal fashion. private VBox fxVBox; //Holds the JavaFX Elements in vertical fashion. private JFXPanel fxPanelCharts; //FX panel to hold the group - private BarChart fxChartEvents; //Yearly/Monthly events - Bar chart + private BarChart fxChartEvents; //Yearly/Monthly events - Bar chart private ScrollPane fxScrollEvents; //Scroll Panes for dealing with oversized an oversized chart private static final int FRAME_HEIGHT = 700; //Sizing constants private static final int FRAME_WIDTH = 1200; private Button fxZoomOutButton; //Navigation buttons private ComboBox fxDropdownSelectYears; //Dropdown box for selecting years. Useful when the charts' scale means some years are unclickable, despite having events. - private final Stack> fxStackPrevCharts = new Stack>(); //Stack for storing drill-up information. - private BarChart fxChartTopLevel; //the topmost chart, used for resetting to default view. + private final Stack> fxStackPrevCharts = new Stack>(); //Stack for storing drill-up information. + private BarChart fxChartTopLevel; //the topmost chart, used for resetting to default view. private DataResultPanel dataResultPanel; private DataContentPanel dataContentPanel; private ProgressHandle progress; @@ -154,7 +152,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, if (coreInstaller != null) { fxInited = coreInstaller.isJavaFxInited(); } - + } //Swing components and JavafX components don't play super well together @@ -280,7 +278,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, fxZoomOutButton.setOnAction(new EventHandler() { @Override public void handle(ActionEvent e) { - BarChart bc; + BarChart bc; if (fxStackPrevCharts.size() == 0) { bc = fxChartTopLevel; } else { @@ -347,7 +345,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, * @param allYears The list of years that have barData from the mactime file * @return BarChart scaled to the year level */ - private BarChart createYearChartWithDrill(final List allYears) { + private BarChart createYearChartWithDrill(final List allYears) { final CategoryAxis xAxis = new CategoryAxis(); //Axes are very specific types. Categorys are strings. final NumberAxis yAxis = new NumberAxis(); final Label l = new Label(""); @@ -362,11 +360,11 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, BarChart.Series se = new BarChart.Series(); if (allYears != null) { for (final YearEpoch ye : allYears) { - se.getData().add(new BarChart.Data(String.valueOf(ye.year), ye.getNumFiles())); + se.getData().add(new BarChart.Data(String.valueOf(ye.year), ye.getNumFiles())); } } bcData.add(se); - + //Note: // BarChart.Data wraps the Java Nodes class. BUT, until a BarChart.Data gets added to an actual series, it's node is null, and you can perform no operations on it. @@ -375,7 +373,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, // But it is for this reason that the chart generating functions have two forloops. I do not believe they can be condensed into a single loop due to the nodes being null until // an undetermined point in time. BarChart bc = new BarChart(xAxis, yAxis, bcData); - for (final BarChart.Data barData : bc.getData().get(0).getData()) { //.get(0) refers to the BarChart.Series class to work on. There is only one series in this graph, so get(0) is safe. + for (final BarChart.Data barData : bc.getData().get(0).getData()) { //.get(0) refers to the BarChart.Series class to work on. There is only one series in this graph, so get(0) is safe. barData.getNode().setScaleX(.5); final javafx.scene.Node barNode = barData.getNode(); @@ -393,7 +391,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, Platform.runLater(new Runnable() { @Override public void run() { - BarChart b = + BarChart b = createMonthsWithDrill(findYear(allYears, Integer.valueOf(barData.getXValue()))); fxChartEvents = b; fxScrollEvents.setContent(fxChartEvents); @@ -416,7 +414,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, * Displays a chart with events from one year only, separated into 1-month chunks. * Always 12 per year, empty months are represented by no bar. */ - private BarChart createMonthsWithDrill(final YearEpoch ye) { + private BarChart createMonthsWithDrill(final YearEpoch ye) { final CategoryAxis xAxis = new CategoryAxis(); final NumberAxis yAxis = new NumberAxis(); @@ -435,7 +433,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, final BarChart bc = new BarChart(xAxis, yAxis, bcData); for (int i = 0; i < 12; i++) { - for (final BarChart.Data barData : bc.getData().get(0).getData()) { + for (final BarChart.Data barData : bc.getData().get(0).getData()) { //Note: // All the charts of this package have a problem where when the chart gets below a certain pixel ratio, the barData stops drawing. The axes and the labels remain, // But the actual chart barData is invisible, unclickable, and unrendered. To partially compensate for that, barData.getNode() can be manually scaled up to increase visibility. @@ -481,22 +479,21 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, * Displays a chart with events from one month only. * Up to 31 days per month, as low as 28 as determined by the specific MonthEpoch */ - private BarChart createEventsByMonth(final MonthEpoch me, final YearEpoch ye) { + private BarChart createEventsByMonth(final MonthEpoch me, final YearEpoch ye) { final CategoryAxis xAxis = new CategoryAxis(); final NumberAxis yAxis = new NumberAxis(); xAxis.setLabel("Day of Month"); yAxis.setLabel("Number of Events"); - ObservableList> bcData - = makeObservableListByMonthAllDays(me, ye.getYear()); - BarChart.Series series = new BarChart.Series(bcData); + ObservableList> bcData = makeObservableListByMonthAllDays(me, ye.getYear()); + BarChart.Series series = new BarChart.Series(bcData); series.setName(me.getMonthName() + " " + ye.getYear()); - ObservableList> ol = + ObservableList> ol = FXCollections.>observableArrayList(series); final BarChart bc = new BarChart(xAxis, yAxis, ol); - for (final BarChart.Data barData : bc.getData().get(0).getData()) { + for (final BarChart.Data barData : bc.getData().get(0).getData()) { //data.getNode().setScaleX(2); final javafx.scene.Node barNode = barData.getNode(); @@ -513,7 +510,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, public void handle(MouseEvent e) { final int day = (Integer.valueOf((barData.getXValue()).split("-")[1])); final DayEpoch de = myme.getDay(day); - final List afs; + final List afs; if (de != null) { afs = de.getEvents(); } else { @@ -541,13 +538,13 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, return bc; } - private static ObservableList> makeObservableListByMonthAllDays(final MonthEpoch me, int year) { - ObservableList> bcData = FXCollections.observableArrayList(); + private static ObservableList> makeObservableListByMonthAllDays(final MonthEpoch me, int year) { + ObservableList> bcData = FXCollections.observableArrayList(); int totalDays = me.getTotalNumDays(year); for (int i = 1; i <= totalDays; ++i) { DayEpoch day = me.getDay(i); int numFiles = day == null ? 0 : day.getNumFiles(); - BarChart.Data d = new BarChart.Data(me.month + 1 + "-" + i, numFiles); + BarChart.Data d = new BarChart.Data(me.month + 1 + "-" + i, numFiles); d.setExtraValue(me); bcData.add(d); } @@ -794,7 +791,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, private class DayEpoch extends Epoch { - private List files = new ArrayList<>(); + private final List fileIds = new ArrayList<>(); int dayNum = 0; //Day of the month this Epoch represents, 1 indexed: 28=28. DayEpoch(int dayOfMonth) { @@ -807,40 +804,49 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, @Override public int getNumFiles() { - return files.size(); + return fileIds.size(); } public void add(AbstractFile af) { - files.add(af); + fileIds.add(af.getId()); } - List getEvents() { - return this.files; + List getEvents() { + return this.fileIds; } } // The node factories used to make lists of files to send to the result viewer - private class FileNodeChildFactory extends ChildFactory { + private class FileNodeChildFactory extends ChildFactory { - List l; + List fileIds; - FileNodeChildFactory(List l) { - this.l = l; + FileNodeChildFactory(List fileIds) { + this.fileIds = fileIds; } @Override - protected boolean createKeys(List list) { - list.addAll(l); + protected boolean createKeys(List list) { + list.addAll(fileIds); return true; } @Override - protected Node createNodeForKey(AbstractFile file) { + protected Node createNodeForKey(Long fileId) { + AbstractFile af = null; + try { + af = skCase.getAbstractFileById(fileId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error getting file by id and creating a node in Timeline: " + fileId, ex); + //no node will be shown for this object + return null; + } + Node wrapped; - if (file.isDir()) { - wrapped = new DirectoryNode(file, false); + if (af.isDir()) { + wrapped = new DirectoryNode(af, false); } else { - wrapped = new FileNode(file, false); + wrapped = new FileNode(af, false); } return new FilterNodeLeaf(wrapped); } @@ -848,8 +854,8 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, private class FileRootNode extends DisplayableItemNode { - FileRootNode(String NAME, List l) { - super(Children.create(new FileNodeChildFactory(l), true)); + FileRootNode(String NAME, List fileIds) { + super(Children.create(new FileNodeChildFactory(fileIds), true)); super.setName(NAME); super.setDisplayName(NAME); } @@ -931,16 +937,11 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, + java.io.File.separator + currentCase.getName() + "-" + datenotime + ".txt"; // Run query to get all files - String filesAndDirs = "name != '.' " + final String filesAndDirs = "name != '.' " + "AND name != '..'"; - List files = null; + List fileIds = null; try { - //TODO this causes one of the Timelines memory limitations - //need another method findAllFileIdsWhere() not to store all file objects in memory - //then create the AbstractFile object as needed later - //OR pass in object reader to optional findAllFilesWhere(query, reader) - //to collect 1 object at a time (much like cursor, but hiding SQL layer) - files = skCase.findAllFilesWhere(filesAndDirs); + fileIds = skCase.findAllFileIdsWhere(filesAndDirs); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error querying image files to make a body file: " + bodyFilePath, ex); return null; @@ -958,7 +959,8 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, BufferedWriter out = null; try { out = new BufferedWriter(fileWriter); - for (AbstractFile file : files) { + for (long fileId : fileIds) { + AbstractFile file = skCase.getAbstractFileById(fileId); // try { // MD5|name|inode|mode_as_string|ObjId|GID|size|atime|mtime|ctime|crtime if (file.getMd5Hash() != null) { @@ -998,6 +1000,10 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, out.write(Long.toString(file.getCrtime())); out.write("\n"); } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error querying file by id", ex); + return null; + } catch (IOException ex) { logger.log(Level.WARNING, "Error while trying to write data to the body file.", ex); return null; @@ -1043,8 +1049,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, } catch (IOException ioe) { logger.log(Level.SEVERE, "Could not create mactime file, encountered error ", ioe); return null; - } - finally { + } finally { if (writer != null) { try { writer.close(); @@ -1053,7 +1058,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, } } } - + return macfile; } From b65d810879d2d79a3b3fce3c1bbe075ada4558fa Mon Sep 17 00:00:00 2001 From: adam-m Date: Mon, 17 Jun 2013 16:25:45 -0400 Subject: [PATCH 08/12] another timeline optimization to avoid unneccessary file object creation when populating the chart from mactime --- .../sleuthkit/autopsy/timeline/Timeline.java | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java index 40e1968599..69f7db0f2a 100644 --- a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java +++ b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java @@ -698,7 +698,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, return month; } - public void add(AbstractFile af, int month, int day) { + public void add(long fileId, int month, int day) { // see if this month is in the list MonthEpoch monthEpoch = null; for (MonthEpoch me : months) { @@ -714,7 +714,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, } // add the file the the MonthEpoch object - monthEpoch.add(af, day); + monthEpoch.add(fileId, day); } } @@ -757,7 +757,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, return de; } - public void add(AbstractFile af, int day) { + public void add(long fileId, int day) { DayEpoch dayEpoch = null; for (DayEpoch de : days) { if (de.getDayInt() == day) { @@ -771,7 +771,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, days.add(dayEpoch); } - dayEpoch.add(af); + dayEpoch.add(fileId); } /** @@ -807,8 +807,8 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, return fileIds.size(); } - public void add(AbstractFile af) { - fileIds.add(af.getId()); + public void add(long fileId) { + fileIds.add(fileId); } List getEvents() { @@ -901,16 +901,8 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, prevYear = year; } - // create and add the file - AbstractFile file; - try { - file = skCase.getAbstractFileById(ObjId); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Could not find a file with ID " + ObjId, ex); - continue; - } if (ye != null) { - ye.add(file, month, day); + ye.add(ObjId, month, day); } } From cea8e4d3377e7f2975a32f0db4224c3a09334e84 Mon Sep 17 00:00:00 2001 From: adam-m Date: Mon, 17 Jun 2013 16:55:42 -0400 Subject: [PATCH 09/12] timeline: use lazy loading rather than background loading of nodes for the day - better for huge number of nodes --- .../sleuthkit/autopsy/timeline/Timeline.java | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java index 69f7db0f2a..d6e17c176b 100644 --- a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java +++ b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java @@ -77,14 +77,13 @@ import org.openide.awt.ActionReferences; import org.openide.awt.ActionRegistration; import org.openide.modules.InstalledFileLocator; import org.openide.modules.ModuleInstall; -import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; import org.openide.nodes.Node; -import org.openide.util.Exceptions; import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; import org.openide.util.actions.Presenter; +import org.openide.util.lookup.Lookups; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponents.DataContentPanel; @@ -99,6 +98,7 @@ import org.sleuthkit.autopsy.datamodel.FileNode; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -817,21 +817,40 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, } // The node factories used to make lists of files to send to the result viewer - private class FileNodeChildFactory extends ChildFactory { + // using the lazy loading (rather than background) loading option to facilitate + // loading a huge number of nodes for the given day + private class FileNodeChildFactory extends Children.Keys { - List fileIds; + private List fileIds; FileNodeChildFactory(List fileIds) { + super(true); this.fileIds = fileIds; } @Override - protected boolean createKeys(List list) { - list.addAll(fileIds); - return true; + protected void addNotify() { + super.addNotify(); + setKeys(fileIds); } @Override + protected void removeNotify() { + super.removeNotify(); + setKeys(new ArrayList()); + } + + @Override + protected Node[] createNodes(Long t) { + return new Node[]{createNodeForKey(t)}; + } + + // @Override + // protected boolean createKeys(List list) { + // list.addAll(fileIds); + // return true; + // } + //@Override protected Node createNodeForKey(Long fileId) { AbstractFile af = null; try { @@ -841,7 +860,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, //no node will be shown for this object return null; } - + Node wrapped; if (af.isDir()) { wrapped = new DirectoryNode(af, false); @@ -855,7 +874,8 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, private class FileRootNode extends DisplayableItemNode { FileRootNode(String NAME, List fileIds) { - super(Children.create(new FileNodeChildFactory(fileIds), true)); + //super(Children.create(new FileNodeChildFactory(fileIds), true)); + super(new FileNodeChildFactory(fileIds), Lookups.singleton(fileIds)); super.setName(NAME); super.setDisplayName(NAME); } From cc17142ca613e783bc27b08c08e2f5a9e11e5e03 Mon Sep 17 00:00:00 2001 From: adam-m Date: Mon, 17 Jun 2013 17:04:39 -0400 Subject: [PATCH 10/12] timeline: reset current nodes before loading nodes for a new day --- .../src/org/sleuthkit/autopsy/timeline/Timeline.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java index d6e17c176b..74aafa2b59 100644 --- a/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java +++ b/Timeline/src/org/sleuthkit/autopsy/timeline/Timeline.java @@ -508,6 +508,15 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, @Override public void handle(MouseEvent e) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + //reset the view and free the current nodes before loading new ones + final FileRootNode d = new FileRootNode("Empty Root", new ArrayList()); + dataResultPanel.setNode(d); + dataResultPanel.setPath("Loading..."); + } + }); final int day = (Integer.valueOf((barData.getXValue()).split("-")[1])); final DayEpoch de = myme.getDay(day); final List afs; @@ -521,7 +530,7 @@ public class Timeline extends CallableSystemAction implements Presenter.Toolbar, SwingUtilities.invokeLater(new Runnable() { @Override public void run() { - final FileRootNode d = new FileRootNode("Test Root", afs); + final FileRootNode d = new FileRootNode("Root", afs); dataResultPanel.setNode(d); //set result viewer title path with the current date String dateString = ye.getYear() + "-" + (1 + me.getMonthInt()) + "-" + +de.dayNum; From 2167f41bbe4800afa0034ddfe5a6b61300819cf4 Mon Sep 17 00:00:00 2001 From: adam-m Date: Tue, 18 Jun 2013 09:42:20 -0400 Subject: [PATCH 11/12] updated NEWS to include yaffs2 and ext4 --- NEWS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.txt b/NEWS.txt index 2ad093db0b..28a9453ec0 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -3,6 +3,7 @@ New features: - Logical files and folders support - New file views in directory tree to view: deleted, executable, archive files and files by size +- ext4 and yaffs2 support (via TSK 4.1.0) Improvements: - Improvements to tagging of files and keyword search results From 3415c49b0411f23ce4ee76bea2885232cc0bee0e Mon Sep 17 00:00:00 2001 From: adam-m Date: Tue, 18 Jun 2013 12:48:39 -0400 Subject: [PATCH 12/12] keyword search: fixes duplicates during ingest for Local Files - when local file set added, ensures to only search the currently ingested sources --- .../autopsy/keywordsearch/Ingester.java | 27 +++++++++-------- .../keywordsearch/KeywordQueryFilter.java | 24 +++++++++------ .../KeywordSearchIngestModule.java | 30 ++++++++----------- 3 files changed, 42 insertions(+), 39 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index a251950ba6..cb2c95799d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -41,6 +41,7 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.SolrInputDocument; import org.openide.util.Exceptions; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.keywordsearch.Server.SolrServerNoPortException; @@ -52,10 +53,10 @@ import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.Directory; import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.FsContent; -import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.ReadContentInputStream; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** @@ -184,6 +185,12 @@ public class Ingester { private class GetContentFieldsV extends ContentVisitor.Default> { + private SleuthkitCase curCase = null; + + GetContentFieldsV() { + curCase = Case.getCurrentCase().getSleuthkitCase(); + } + @Override protected Map defaultVisit(Content cntnt) { return new HashMap(); @@ -217,11 +224,7 @@ public class Ingester { @Override public Map visit(LocalFile lf) { - final Map params = new HashMap(); - params.put(Server.Schema.ID.toString(), Long.toString(lf.getId())); - params.put(Server.Schema.FILE_NAME.toString(), lf.getName()); - params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1)); - return params; + return getCommonFields(lf); } private Map getCommonFsContentFields(Map params, FsContent fsContent) { @@ -235,15 +238,13 @@ public class Ingester { private Map getCommonFields(AbstractFile af) { Map params = new HashMap(); params.put(Server.Schema.ID.toString(), Long.toString(af.getId())); - long imageId = -1; + long dataSourceId = -1; try { - Image image = af.getImage(); - if (image != null) { - imageId = image.getId(); - } - params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(imageId)); + dataSourceId = curCase.getFileDataSource(af); + params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(dataSourceId)); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Could not get image id to properly index the file " + af.getId()); + logger.log(Level.SEVERE, "Could not get data source id to properly index the file " + af.getId()); + params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1)); } params.put(Server.Schema.FILE_NAME.toString(), af.getName()); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java index a62b884ae8..1afbc464a9 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordQueryFilter.java @@ -18,6 +18,10 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + /** * * Filter to restrict query only specific files, chunks, images @@ -27,23 +31,23 @@ public class KeywordQueryFilter { public static enum FilterType { - FILE, CHUNK, IMAGE + FILE, CHUNK, DATA_SOURCE }; - private long[] idFilters; + private SetidFilters; private FilterType filterType; public KeywordQueryFilter(FilterType filterType, long id) { this.filterType = filterType; - this.idFilters = new long[1]; - this.idFilters[0] = id; + this.idFilters = new HashSet(); + this.idFilters.add(id); } - public KeywordQueryFilter(FilterType filterType, long[] ids) { + public KeywordQueryFilter(FilterType filterType, Setids) { this.filterType = filterType; this.idFilters = ids; } - public long[] getIdFilters() { + public Set getIdFilters() { return idFilters; } @@ -55,12 +59,14 @@ public class KeywordQueryFilter { public String toString() { StringBuilder sb = new StringBuilder(); String id = null; - for (int i = 0; i < idFilters.length; ++i) { + + Iteratorit = idFilters.iterator(); + for (int i = 0; it.hasNext(); ++i) { if (i > 0) { sb.append(" "); //OR } - long idVal = idFilters[i]; - if (filterType == FilterType.IMAGE) { + long idVal = it.next(); + if (filterType == FilterType.DATA_SOURCE) { id = Server.Schema.IMAGE_ID.toString(); } else { id = Server.Schema.ID.toString(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index c6d3538fc8..001ae393b0 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -44,6 +44,7 @@ import org.netbeans.api.progress.aggregate.AggregateProgressFactory; import org.netbeans.api.progress.aggregate.AggregateProgressHandle; import org.netbeans.api.progress.aggregate.ProgressContributor; import org.openide.util.Cancellable; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.StopWatch; @@ -60,6 +61,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.SleuthkitCase; @@ -115,7 +117,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { private Map> currentResults; //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread - private Set curImageIds; + private Set curDataSourceIds; private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy private static final Lock searcherLock = rwLock.writeLock(); private volatile int messageID = 0; @@ -128,6 +130,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { private boolean initialized = false; private KeywordSearchConfigurationPanel panel; private Tika tikaFormatDetector; + private enum IngestStatus { @@ -160,12 +163,10 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { return ProcessResult.OK; } try { - //add image id of the file to the set, keeping track of images being ingested - final Image fileImage = abstractFile.getImage(); - if (fileImage != null) { - //not all Content objects have an image associated (e.g. LocalFiles) - curImageIds.add(fileImage.getId()); - } + //add data source id of the file to the set, keeping track of images being ingested + final long fileSourceId = caseHandle.getFileDataSource(abstractFile); + curDataSourceIds.add(fileSourceId); + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex); } @@ -288,7 +289,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { private void cleanup() { ingestStatus.clear(); currentResults.clear(); - curImageIds.clear(); + curDataSourceIds.clear(); currentSearcher = null; //finalSearcher = null; //do not collect, might be finalizing @@ -399,7 +400,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { //keeps track of all results per run not to repeat reporting the same hits currentResults = new HashMap>(); - curImageIds = new HashSet(); + curDataSourceIds = new HashSet(); indexer = new Indexer(); @@ -930,15 +931,10 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { del = new TermComponentQuery(keywordQuery); } - //limit search to currently ingested images - final long imageIds[] = new long[curImageIds.size()]; - final Iterator it = curImageIds.iterator(); - for (int imageI = 0; it.hasNext(); ++imageI) { - imageIds[imageI] = it.next(); - } + //limit search to currently ingested data sources //set up a filter with 1 or more image ids OR'ed - final KeywordQueryFilter imageFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.IMAGE, imageIds); - del.addFilter(imageFilter); + final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds); + del.addFilter(dataSourceFilter); Map> queryResult = null;