From b82d0a2e9cc3a4a43f74f4353793508dfaf90a04 Mon Sep 17 00:00:00 2001 From: jmillman Date: Thu, 12 Nov 2015 15:37:39 -0500 Subject: [PATCH] return and cache event stripes at the DB layer --- .../timeline/datamodel/EventStripe.java | 17 ++++ .../datamodel/FilteredEventsModel.java | 4 +- .../autopsy/timeline/db/EventDB.java | 17 +++- .../autopsy/timeline/db/EventsRepository.java | 6 +- .../ui/AbstractVisualizationPane.java | 2 +- .../ui/detailview/DetailViewPane.java | 20 ++--- .../timeline/ui/detailview/EventAxis.java | 18 ++-- .../ui/detailview/EventClusterNode.java | 6 +- .../ui/detailview/EventDetailsChart.java | 83 +++++++++---------- 9 files changed, 98 insertions(+), 75 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/EventStripe.java b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/EventStripe.java index cd457d336e..0b772cc6a2 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/EventStripe.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/EventStripe.java @@ -83,6 +83,22 @@ public final class EventStripe implements EventBundle { */ private final Set hashHits = new HashSet<>(); + public EventStripe withParent(EventCluster parent) { + EventStripe eventStripe = new EventStripe(parent, this.type, this.description, this.lod); + eventStripe.clusters.addAll(clusters); + eventStripe.eventIDs.addAll(eventIDs); + eventStripe.tagged.addAll(tagged); + eventStripe.hashHits.addAll(hashHits); + return eventStripe; + } + + private EventStripe(EventCluster parent, EventType type, String description, DescriptionLoD lod) { + this.parent = parent; + this.type = type; + this.description = description; + this.lod = lod; + } + public EventStripe(EventCluster cluster, EventCluster parent) { clusters.add(cluster); @@ -158,4 +174,5 @@ public final class EventStripe implements EventBundle { public SortedSet< EventCluster> getClusters() { return Collections.unmodifiableSortedSet(clusters); } + } diff --git a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java index c11bb97012..b658e40c6a 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java @@ -327,7 +327,7 @@ public final class FilteredEventsModel { * @return a list of event clusters at the requested zoom levels that are * within the requested time range and pass the requested filter */ - public List getEventClusters() { + public List getEventClusters() { final Interval range; final RootFilter filter; final EventTypeZoomLevel zoom; @@ -348,7 +348,7 @@ public final class FilteredEventsModel { * range and pass the requested filter, using the given aggregation * to control the grouping of events */ - public List getEventClusters(ZoomParams params) { + public List getEventClusters(ZoomParams params) { return repo.getEventClusters(params); } diff --git a/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java b/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java index 1c1d3e3ab7..0c65ef7e98 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/db/EventDB.java @@ -48,6 +48,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.joda.time.DateTimeZone; import org.joda.time.Interval; import org.joda.time.Period; @@ -55,6 +56,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.timeline.TimeLineController; import org.sleuthkit.autopsy.timeline.datamodel.EventCluster; +import org.sleuthkit.autopsy.timeline.datamodel.EventStripe; import org.sleuthkit.autopsy.timeline.datamodel.TimeLineEvent; import org.sleuthkit.autopsy.timeline.datamodel.eventtype.BaseTypes; import org.sleuthkit.autopsy.timeline.datamodel.eventtype.EventType; @@ -1062,7 +1064,7 @@ public class EventDB { * the supplied filter, aggregated according to the given event type * and description zoom levels */ - List getClusteredEvents(ZoomParams params) { + List getClusteredEvents(ZoomParams params) { //unpack params Interval timeRange = params.getTimeRange(); RootFilter filter = params.getFilter(); @@ -1105,7 +1107,6 @@ public class EventDB { try (Statement createStatement = con.createStatement(); ResultSet rs = createStatement.executeQuery(query)) { while (rs.next()) { - events.add(eventClusterHelper(rs, useSubTypes, descriptionLOD, filter.getTagsFilter())); } } catch (SQLException ex) { @@ -1114,7 +1115,17 @@ public class EventDB { DBLock.unlock(); } - return mergeEventClusters(rangeInfo.getPeriodSize().getPeriod(), events); + List mergeEventClusters = mergeEventClusters(rangeInfo.getPeriodSize().getPeriod(), events); + + //merge clusters to stripes + Map, EventStripe> stripeDescMap = new HashMap<>(); + + for (EventCluster eventCluster : mergeEventClusters) { + stripeDescMap.merge(ImmutablePair.of(eventCluster.getEventType(), eventCluster.getDescription()), + new EventStripe(eventCluster, null), EventStripe::merge); + } + + return stripeDescMap.values().stream().sorted(Comparator.comparing(EventStripe::getStartMillis)).collect(Collectors.toList()); } /** diff --git a/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java b/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java index 05d3551e05..ff20bd5df0 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java @@ -53,10 +53,8 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ThreadConfined; -import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.timeline.CancellationProgressTask; import org.sleuthkit.autopsy.timeline.TimeLineController; -import org.sleuthkit.autopsy.timeline.datamodel.EventCluster; import org.sleuthkit.autopsy.timeline.datamodel.FilteredEventsModel; import org.sleuthkit.autopsy.timeline.datamodel.TimeLineEvent; import org.sleuthkit.autopsy.timeline.datamodel.eventtype.ArtifactEventType; @@ -106,7 +104,7 @@ public class EventsRepository { private final LoadingCache minCache; private final LoadingCache idToEventCache; private final LoadingCache> eventCountsCache; - private final LoadingCache> eventClusterCache; + private final LoadingCache> eventClusterCache; private final ObservableMap datasourcesMap = FXCollections.observableHashMap(); private final ObservableMap hashSetMap = FXCollections.observableHashMap(); @@ -213,7 +211,7 @@ public class EventsRepository { } - synchronized public List getEventClusters(ZoomParams params) { + synchronized public List getEventClusters(ZoomParams params) { return eventClusterCache.getUnchecked(params); } diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/AbstractVisualizationPane.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/AbstractVisualizationPane.java index e870b064f7..9d2f3bfb0f 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/AbstractVisualizationPane.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/AbstractVisualizationPane.java @@ -252,7 +252,7 @@ public abstract class AbstractVisualizationPane } }); - update(); +// update(); } @Subscribe diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/DetailViewPane.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/DetailViewPane.java index 5517a0d415..ade7d4580e 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/DetailViewPane.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/DetailViewPane.java @@ -66,7 +66,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.timeline.FXMLConstructor; import org.sleuthkit.autopsy.timeline.TimeLineController; import org.sleuthkit.autopsy.timeline.datamodel.EventBundle; -import org.sleuthkit.autopsy.timeline.datamodel.EventCluster; +import org.sleuthkit.autopsy.timeline.datamodel.EventStripe; import org.sleuthkit.autopsy.timeline.datamodel.FilteredEventsModel; import org.sleuthkit.autopsy.timeline.datamodel.eventtype.EventType; import org.sleuthkit.autopsy.timeline.ui.AbstractVisualizationPane; @@ -85,7 +85,7 @@ import org.sleuthkit.autopsy.timeline.zooming.DescriptionLoD; * EventTypeMap, and dataSets is all linked directly to the ClusterChart which * must only be manipulated on the JavaFx thread. */ -public class DetailViewPane extends AbstractVisualizationPane, EventDetailsChart> { +public class DetailViewPane extends AbstractVisualizationPane, EventDetailsChart> { private final static Logger LOGGER = Logger.getLogger(DetailViewPane.class.getName()); @@ -93,7 +93,7 @@ public class DetailViewPane extends AbstractVisualizationPane verticalAxis = new EventAxis(); + private final Axis verticalAxis = new EventAxis(); private final ScrollBar vertScrollBar = new ScrollBar(); private final Region scrollBarSpacer = new Region(); @@ -101,7 +101,7 @@ public class DetailViewPane extends AbstractVisualizationPane> highlightedNodes = FXCollections.synchronizedObservableList(FXCollections.observableArrayList()); //private access to barchart data - private final Map> eventTypeToSeriesMap = new ConcurrentHashMap<>(); + private final Map> eventTypeToSeriesMap = new ConcurrentHashMap<>(); public ObservableList> getEventBundles() { return chart.getEventBundles(); @@ -224,7 +224,7 @@ public class DetailViewPane extends AbstractVisualizationPane getYAxis() { + protected Axis getYAxis() { return verticalAxis; } @@ -253,9 +253,9 @@ public class DetailViewPane extends AbstractVisualizationPane getSeries(final EventType et) { + private XYChart.Series getSeries(final EventType et) { return eventTypeToSeriesMap.computeIfAbsent(et, (EventType t) -> { - XYChart.Series series = new XYChart.Series<>(); + XYChart.Series series = new XYChart.Series<>(); series.setName(et.getDisplayName()); // Platform.runLater(() -> { dataSeries.add(series); @@ -310,7 +310,7 @@ public class DetailViewPane extends AbstractVisualizationPane eventClusters = filteredEvents.getEventClusters(); + List eventClusters = filteredEvents.getEventClusters(); final int size = eventClusters.size(); updateMessage(NbBundle.getMessage(this.getClass(), "DetailViewPane.loggedTask.updateUI")); @@ -318,9 +318,9 @@ public class DetailViewPane extends AbstractVisualizationPane xyData = new BarChart.Data<>(new DateTime(cluster.getStartMillis()), cluster); + final XYChart.Data xyData = new BarChart.Data<>(new DateTime(cluster.getStartMillis()), cluster); // Platform.runLater(() -> { getSeries(cluster.getEventType()).getData().add(xyData); diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventAxis.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventAxis.java index 89b4de0ec7..544a5e9dd9 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventAxis.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventAxis.java @@ -22,21 +22,21 @@ import java.util.Collections; import java.util.List; import javafx.scene.chart.Axis; import javafx.scene.chart.XYChart; -import org.sleuthkit.autopsy.timeline.datamodel.EventCluster; +import org.sleuthkit.autopsy.timeline.datamodel.EventStripe; /** * No-Op axis that doesn't do anything usefull but is necessary to pass * AggregateEvent as the second member of {@link XYChart.Data} objects */ -class EventAxis extends Axis { +class EventAxis extends Axis { @Override - public double getDisplayPosition(EventCluster value) { + public double getDisplayPosition(EventStripe value) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override - public EventCluster getValueForDisplay(double displayPosition) { + public EventStripe getValueForDisplay(double displayPosition) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @@ -46,17 +46,17 @@ class EventAxis extends Axis { } @Override - public boolean isValueOnAxis(EventCluster value) { + public boolean isValueOnAxis(EventStripe value) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override - public double toNumericValue(EventCluster value) { + public double toNumericValue(EventStripe value) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override - public EventCluster toRealValue(double value) { + public EventStripe toRealValue(double value) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @@ -66,7 +66,7 @@ class EventAxis extends Axis { } @Override - protected List calculateTickValues(double length, Object range) { + protected List calculateTickValues(double length, Object range) { return Collections.emptyList(); } @@ -76,7 +76,7 @@ class EventAxis extends Axis { } @Override - protected String getTickMarkLabel(EventCluster value) { + protected String getTickMarkLabel(EventStripe value) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java index 02ff99d689..7f6ff32816 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java @@ -169,7 +169,7 @@ final public class EventClusterNode extends EventBundleNodeBase call() throws Exception { - Collection bundles; + Collection bundles = null; DescriptionLoD next = loadedDescriptionLoD; do { loadedDescriptionLoD = next; @@ -177,8 +177,8 @@ final public class EventClusterNode extends EventBundleNodeBase new EventStripe(eventCluster, getEventCluster()), //value + .collect(Collectors.toMap(EventStripe::getDescription, //key + eventStripe -> eventStripe.withParent(getEventCluster()), //value EventStripe::merge) //merge method ).values(); next = loadedDescriptionLoD.withRelativeDetail(relativeDetail); diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventDetailsChart.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventDetailsChart.java index 3ce076c544..e311e9a822 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventDetailsChart.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventDetailsChart.java @@ -59,7 +59,6 @@ import javafx.scene.image.ImageView; import javafx.scene.input.MouseEvent; import javafx.scene.shape.Line; import javafx.scene.shape.StrokeLineCap; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.controlsfx.control.action.Action; import org.controlsfx.control.action.ActionUtils; import org.joda.time.DateTime; @@ -71,7 +70,6 @@ import org.sleuthkit.autopsy.timeline.datamodel.EventBundle; import org.sleuthkit.autopsy.timeline.datamodel.EventCluster; import org.sleuthkit.autopsy.timeline.datamodel.EventStripe; import org.sleuthkit.autopsy.timeline.datamodel.FilteredEventsModel; -import org.sleuthkit.autopsy.timeline.datamodel.eventtype.EventType; import org.sleuthkit.autopsy.timeline.filters.AbstractFilter; import org.sleuthkit.autopsy.timeline.filters.DescriptionFilter; import org.sleuthkit.autopsy.timeline.ui.AbstractVisualizationPane; @@ -93,7 +91,7 @@ import org.sleuthkit.autopsy.timeline.zooming.DescriptionLoD; * * //TODO: refactor the projected lines to a separate class. -jm */ -public final class EventDetailsChart extends XYChart implements TimeLineChart { +public final class EventDetailsChart extends XYChart implements TimeLineChart { private static final String styleSheet = GuideLine.class.getResource("EventsDetailsChart.css").toExternalForm(); private static final Image HIDE = new Image("/org/sleuthkit/autopsy/timeline/images/eye--minus.png"); // NON-NLS @@ -145,7 +143,7 @@ public final class EventDetailsChart extends XYChart imp */ private final Group nodeGroup = new Group(); private final ObservableList> bundles = FXCollections.observableArrayList(); - private final Map, EventStripe> stripeDescMap = new ConcurrentHashMap<>(); +// private final Map, EventStripe> stripeDescMap = new ConcurrentHashMap<>(); private final Map stripeNodeMap = new ConcurrentHashMap<>(); private final Map projectionMap = new ConcurrentHashMap<>(); @@ -181,7 +179,7 @@ public final class EventDetailsChart extends XYChart imp */ final SimpleDoubleProperty truncateWidth = new SimpleDoubleProperty(200.0); - EventDetailsChart(TimeLineController controller, DateAxis dateAxis, final Axis verticalAxis, ObservableList> selectedNodes) { + EventDetailsChart(TimeLineController controller, DateAxis dateAxis, final Axis verticalAxis, ObservableList> selectedNodes) { super(dateAxis, verticalAxis); this.controller = controller; @@ -321,55 +319,56 @@ public final class EventDetailsChart extends XYChart imp } @Override - protected synchronized void dataItemAdded(Series series, int i, Data data) { - final EventCluster eventCluster = data.getYValue(); - - EventStripe eventStripe = stripeDescMap.merge(ImmutablePair.of(eventCluster.getEventType(), eventCluster.getDescription()), - new EventStripe(eventCluster, null), - (EventStripe u, EventStripe v) -> { - EventStripeNode removeU = stripeNodeMap.remove(u); - EventStripeNode removeV = stripeNodeMap.remove(v); - Platform.runLater(() -> { - nodeGroup.getChildren().remove(removeU); - nodeGroup.getChildren().remove(removeV); - }); - return EventStripe.merge(u, v); - } - ); + protected synchronized void dataItemAdded(Series series, int i, Data data) { + final EventStripe eventStripe = data.getYValue(); +// +// EventStripe eventStripe = stripeDescMap.put(ImmutablePair.of(eventCluster.getEventType(), eventCluster.getDescription()), eventCluster); +//// new EventStripe(eventCluster, null), +//// (EventStripe u, EventStripe v) -> { +//// EventStripeNode removeU = stripeNodeMap.remove(u); +//// EventStripeNode removeV = stripeNodeMap.remove(v); +//// Platform.runLater(() -> { +//// nodeGroup.getChildren().remove(removeU); +//// nodeGroup.getChildren().remove(removeV); +//// }); +//// return EventStripe.merge(u, v); +//// } +//// ); EventStripeNode stripeNode = new EventStripeNode(EventDetailsChart.this, eventStripe, null); stripeNodeMap.put(eventStripe, stripeNode); Platform.runLater(() -> { - bundles.add(eventCluster); + bundles.add(eventStripe); nodeGroup.getChildren().add(stripeNode); data.setNode(stripeNode); }); } @Override - protected void dataItemChanged(Data data) { + protected void dataItemChanged(Data data) { //TODO: can we use this to help with local detail level adjustment -jm throw new UnsupportedOperationException("Not supported yet."); // NON-NLS //To change body of generated methods, choose Tools | Templates. } @Override - protected void dataItemRemoved(Data data, Series series) { - EventCluster eventCluster = data.getYValue(); - Platform.runLater(() -> { - bundles.removeAll(eventCluster); - }); + protected synchronized void dataItemRemoved(Data data, Series series) { + EventStripe removedStripe = data.getYValue(); +// Platform.runLater(() -> { +// bundles.removeAll(removedStripe); +// }); - EventStripe removedStripe = stripeDescMap.remove(ImmutablePair.of(eventCluster.getEventType(), eventCluster.getDescription())); - if (removedStripe != null) { - EventStripeNode removedNode = stripeNodeMap.remove(removedStripe); - Platform.runLater(() -> { - nodeGroup.getChildren().remove(removedNode); - data.setNode(null); - }); - } +// EventStripe removedStripe = stripeDescMap.remove(ImmutablePair.of(eventCluster.getEventType(), eventCluster.getDescription())); +// if (removedStripe != null) { + EventStripeNode removedNode = stripeNodeMap.remove(removedStripe); + Platform.runLater(() -> { + bundles.removeAll(removedStripe); + nodeGroup.getChildren().remove(removedNode); + data.setNode(null); + }); +// } } @Override - protected void layoutPlotChildren() { + protected synchronized void layoutPlotChildren() { setCursor(Cursor.WAIT); maxY.set(0); if (bandByType.get()) { @@ -393,16 +392,16 @@ public final class EventDetailsChart extends XYChart imp } @Override - protected void seriesAdded(Series series, int i) { + protected void seriesAdded(Series series, int i) { for (int j = 0; j < series.getData().size(); j++) { dataItemAdded(series, j, series.getData().get(j)); } } @Override - protected void seriesRemoved(Series series) { - for (int j = 0; j < series.getData().size(); j++) { - dataItemRemoved(series.getData().get(j), series); + protected void seriesRemoved(Series series) { + for (Data data : series.getData()) { + dataItemRemoved(data, series); } } @@ -413,7 +412,7 @@ public final class EventDetailsChart extends XYChart imp /** * @return all the nodes that pass the given predicate */ - Iterable> getNodes(Predicate> p) { + synchronized Iterable> getNodes(Predicate> p) { //use this recursive function to flatten the tree of nodes into an iterable. Function, Stream>> stripeFlattener = new Function, Stream>>() { @@ -535,8 +534,6 @@ public final class EventDetailsChart extends XYChart imp return localMax; //return new max } - - private void bundleLayoutHelper(final EventBundleNodeBase bundleNode) { //make sure it is shown bundleNode.setVisible(true);