From 9a06f0448ed18981e7b6f1f97ea4e81c0b2834fb Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 29 Jul 2015 17:38:55 -0400 Subject: [PATCH 1/4] Synchronize an IngestTasksScheduler method for clarity and safety --- Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index f00248ecef..505a343057 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -469,7 +469,7 @@ final class IngestTasksScheduler { * @param taskQueue The queue from which to remove the tasks. * @param jobId The id of the job for which the tasks are to be removed. */ - private void removeTasksForJob(Collection taskQueue, long jobId) { + synchronized private void removeTasksForJob(Collection taskQueue, long jobId) { Iterator iterator = taskQueue.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); From 14a3e3f774f98aa604b5bb98a6d80cdde188c9b2 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 29 Jul 2015 17:59:16 -0400 Subject: [PATCH 2/4] Added listener to VolumeNode so that we can detect when carved files are added. --- .../datamodel/AbstractAbstractFileNode.java | 25 +++++++--- .../autopsy/datamodel/VolumeNode.java | 47 +++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 21ac5abea0..87a3b4ca30 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -57,6 +57,8 @@ public abstract class AbstractAbstractFileNode extends A if (dotIndex > 0) { String ext = name.substring(dotIndex).toLowerCase(); + // If this is an archive file we will listen for ingest events + // that will notify us when new content has been identified. for (String s : FileTypeExtensions.getArchiveExtensions()) { if (ext.equals(s)) { IngestManager.getInstance().addIngestModuleEventListener(pcl); @@ -66,10 +68,10 @@ public abstract class AbstractAbstractFileNode extends A } - private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { + protected final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { String eventType = evt.getPropertyName(); - // See if the new file is a child of ours + // Is this a content changed event? if (eventType.equals(IngestManager.IngestModuleEvent.CONTENT_CHANGED.toString())) { if ((evt.getOldValue() instanceof ModuleContentEvent) == false) { return; @@ -79,16 +81,25 @@ public abstract class AbstractAbstractFileNode extends A return; } Content newContent = (Content) moduleContentEvent.getSource(); + + // Does the event indicate that content has been added to *this* file? if (getContent().getId() == newContent.getId()) { - Children parentsChildren = getParentNode().getChildren(); - if (parentsChildren != null) { - ((ContentChildren)parentsChildren).refreshChildren(); - parentsChildren.getNodesCount(); + // If so, refresh our children. + try { + Children parentsChildren = getParentNode().getChildren(); + if (parentsChildren != null) { + ((ContentChildren)parentsChildren).refreshChildren(); + parentsChildren.getNodesCount(); + } } + catch (NullPointerException ex) { + // Skip + } + } } }; - + // Note: this order matters for the search result, changed it if the order of property headers on the "KeywordSearchNode"changed public static enum AbstractFilePropertyType { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java index 36bda58844..0160b120af 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java @@ -18,13 +18,21 @@ */ package org.sleuthkit.autopsy.datamodel; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; import java.util.ArrayList; import java.util.List; import javax.swing.Action; +import org.openide.nodes.Children; import org.openide.nodes.Sheet; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.directorytree.ExplorerNodeActionVisitor; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; +import org.sleuthkit.autopsy.ingest.IngestManager; +import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.VirtualDirectory; import org.sleuthkit.datamodel.Volume; /** @@ -59,8 +67,47 @@ public class VolumeNode extends AbstractContentNode { this.setDisplayName(tempVolName); this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/vol-icon.png"); //NON-NLS + // Listen for ingest events so that we can detect new added files (e.g. carved) + IngestManager.getInstance().addIngestModuleEventListener(pcl); + } + protected final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { + String eventType = evt.getPropertyName(); + + // See if the new file is a child of ours + if (eventType.equals(IngestManager.IngestModuleEvent.CONTENT_CHANGED.toString())) { + if ((evt.getOldValue() instanceof ModuleContentEvent) == false) { + return; + } + ModuleContentEvent moduleContentEvent = (ModuleContentEvent) evt.getOldValue(); + if ((moduleContentEvent.getSource() instanceof Content) == false) { + return; + } + Content newContent = (Content) moduleContentEvent.getSource(); + + try { + Content parent = newContent.getParent(); + if (parent != null) { + // Is this a new carved file? + if (parent.getName().equals(VirtualDirectory.NAME_CARVED)) { + // Was this new carved file produced from this volume? + if (parent.getParent().getId() == getContent().getId()) { + Children children = getChildren(); + if (children != null) { + ((ContentChildren)children).refreshChildren(); + children.getNodesCount(); + } + } + } + } + } + catch (TskCoreException ex) { + // Do nothing. + } + } + }; + /** * Right click action for volume node * From 0bb948cefe45807456ae8629a4ed48348fb9403c Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 29 Jul 2015 18:09:55 -0400 Subject: [PATCH 3/4] Made property change listeners private instead of protected. --- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 2 +- Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 87a3b4ca30..bd133ab4c2 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -68,7 +68,7 @@ public abstract class AbstractAbstractFileNode extends A } - protected final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { + private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { String eventType = evt.getPropertyName(); // Is this a content changed event? diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java index 0160b120af..18bc5f9097 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java @@ -72,7 +72,7 @@ public class VolumeNode extends AbstractContentNode { } - protected final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { + private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { String eventType = evt.getPropertyName(); // See if the new file is a child of ours From d74e3d2a4bab0d1164d6418de620145e279ed274 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 30 Jul 2015 09:15:02 -0400 Subject: [PATCH 4/4] Eliminate thread-unsafe flush of concurrent queues in ingest tasks scheduler --- .../autopsy/ingest/IngestTasksScheduler.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 505a343057..20612d7a6a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -248,19 +248,27 @@ final class IngestTasksScheduler { } /** - * Clears the task scheduling queues for an ingest job, but does nothing - * about tasks that have already been taken by ingest threads. Those tasks - * will be flushed out when the ingest threads call back with their task - * completed notifications. + * Clears the "upstream" task scheduling queues for an ingest job, but does + * nothing about tasks that have already been shuffled into the concurrently + * accessed blocking queues shared with the ingest threads. Note that tasks + * in the "downstream" queues or already taken by the ingest threads will be + * flushed out when the ingest threads call back with their task completed + * notifications. * * @param job The job for which the tasks are to to canceled. */ synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { + /** + * This code should not flush the blocking queues that are concurrently + * accessed by the ingest threads. This is because the "lock striping" + * and "weakly consistent" iterators of these collections make it so + * that this code could have a different view of the queues than the + * ingest threads. It does clean out the directory level tasks before + * they are exploded into file tasks. + */ long jobId = job.getId(); this.removeTasksForJob(this.rootDirectoryTasks, jobId); this.removeTasksForJob(this.directoryTasks, jobId); - this.removeTasksForJob(this.pendingFileTasks, jobId); - this.removeTasksForJob(this.pendingDataSourceTasks, jobId); this.shuffleFileTaskQueues(); }