From bd1e9a58d83fa5299736cb07a89742308c7a027d Mon Sep 17 00:00:00 2001 From: jmillman Date: Thu, 17 Sep 2015 14:06:29 -0400 Subject: [PATCH] fix description generation and clustering - rename getAggregatredEvents to getEventClusters - make more members private - improve logic to get subclusters - introduce notion of RelativeDetail for navigating DescriptionLoDs --- .../datamodel/FilteredEventsModel.java | 6 +- .../autopsy/timeline/db/EventsRepository.java | 12 +- .../ui/detailview/AbstractDetailViewNode.java | 178 ++++++++++++------ .../ui/detailview/EventClusterNode.java | 12 +- .../ui/detailview/EventStripeNode.java | 10 +- .../timeline/zooming/DescriptionLOD.java | 26 ++- 6 files changed, 160 insertions(+), 84 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java index 80aa51d571..664ae74ddd 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/datamodel/FilteredEventsModel.java @@ -337,7 +337,7 @@ public final class FilteredEventsModel { zoom = requestedTypeZoom.get(); lod = requestedLOD.get(); } - return repo.getAggregatedEvents(new ZoomParams(range, zoom, filter, lod)); + return repo.getEventClusters(new ZoomParams(range, zoom, filter, lod)); } /** @@ -347,8 +347,8 @@ public final class FilteredEventsModel { * range and pass the requested filter, using the given aggregation * to control the grouping of events */ - public List getAggregatedEvents(ZoomParams params) { - return repo.getAggregatedEvents(params); + public List getEventClusters(ZoomParams params) { + return repo.getEventClusters(params); } synchronized public boolean handleContentTagAdded(ContentTagAddedEvent evt) { diff --git a/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java b/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java index 6248b99a97..c2a03545a6 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/db/EventsRepository.java @@ -98,7 +98,7 @@ public class EventsRepository { private final LoadingCache idToEventCache; private final LoadingCache> eventCountsCache; - private final LoadingCache> aggregateEventsCache; + private final LoadingCache> eventClusterCache; private final ObservableMap datasourcesMap = FXCollections.observableHashMap(); private final ObservableMap hashSetMap = FXCollections.observableHashMap(); @@ -146,7 +146,7 @@ public class EventsRepository { .maximumSize(1000L) .expireAfterAccess(10, TimeUnit.MINUTES) .build(CacheLoader.from(eventDB::countEventsByType)); - aggregateEventsCache = CacheBuilder.newBuilder() + eventClusterCache = CacheBuilder.newBuilder() .maximumSize(1000L) .expireAfterAccess(10, TimeUnit.MINUTES ).build(CacheLoader.from(eventDB::getClusteredEvents)); @@ -206,8 +206,8 @@ public class EventsRepository { } - synchronized public List getAggregatedEvents(ZoomParams params) { - return aggregateEventsCache.getUnchecked(params); + synchronized public List getEventClusters(ZoomParams params) { + return eventClusterCache.getUnchecked(params); } synchronized public Map countEvents(ZoomParams params) { @@ -218,7 +218,7 @@ public class EventsRepository { minCache.invalidateAll(); maxCache.invalidateAll(); eventCountsCache.invalidateAll(); - aggregateEventsCache.invalidateAll(); + eventClusterCache.invalidateAll(); idToEventCache.invalidateAll(); } @@ -292,7 +292,7 @@ public class EventsRepository { synchronized private void invalidateCaches(Set updatedEventIDs) { eventCountsCache.invalidateAll(); - aggregateEventsCache.invalidateAll(); + eventClusterCache.invalidateAll(); idToEventCache.invalidateAll(updatedEventIDs); try { tagNames.setAll(autoCase.getSleuthkitCase().getTagNamesInUse()); diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AbstractDetailViewNode.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AbstractDetailViewNode.java index 0a720564c9..e52cf31217 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AbstractDetailViewNode.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/AbstractDetailViewNode.java @@ -1,14 +1,29 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2015 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.timeline.ui.detailview; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.Objects.nonNull; import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.stream.Collectors; @@ -61,6 +76,7 @@ import org.sleuthkit.autopsy.timeline.filters.DescriptionFilter; import org.sleuthkit.autopsy.timeline.filters.RootFilter; import org.sleuthkit.autopsy.timeline.filters.TypeFilter; import org.sleuthkit.autopsy.timeline.zooming.DescriptionLOD; +import org.sleuthkit.autopsy.timeline.zooming.EventTypeZoomLevel; import org.sleuthkit.autopsy.timeline.zooming.ZoomParams; import org.sleuthkit.datamodel.SleuthkitCase; @@ -89,11 +105,11 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A b.setVisible(show); b.setManaged(show); } - Map dropShadowMap = new HashMap<>(); + private final Map dropShadowMap = new HashMap<>(); final Color evtColor; private final S parentNode; - DescriptionVisibility descrVis; + private DescriptionVisibility descrVis; /** * Pane that contains AggregateEventNodes of any 'subevents' if they are @@ -102,7 +118,11 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A * //TODO: move more of the control of subnodes/events here and out of * EventDetail Chart */ - final Pane subNodePane = new Pane(); + private final Pane subNodePane = new Pane(); + + Pane getSubNodePane() { + return subNodePane; + } /** * The ImageView used to show the icon for this node's event's type @@ -121,16 +141,28 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A private final T eventBundle; private final EventDetailChart chart; - final SleuthkitCase sleuthkitCase; - final FilteredEventsModel eventsModel; + private final SleuthkitCase sleuthkitCase; - final Button plusButton; - final Button minusButton; + SleuthkitCase getSleuthkitCase() { + return sleuthkitCase; + } - final SimpleObjectProperty descLOD = new SimpleObjectProperty<>(); + FilteredEventsModel getEventsModel() { + return eventsModel; + } + private final FilteredEventsModel eventsModel; + + private final Button plusButton; + private final Button minusButton; + + private final SimpleObjectProperty descLOD = new SimpleObjectProperty<>(); final HBox header; - final Region spacer = new Region(); + Region getSpacer() { + return spacer; + } + + private final Region spacer = new Region(); private final CollapseClusterAction collapseClusterAction; private final ExpandClusterAction expandClusterAction; @@ -160,9 +192,7 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A configureLODButton(minusButton); HBox.setHgrow(spacer, Priority.ALWAYS); - header = new HBox(getDescrLabel(), getCountLabel(), hashIV, tagIV, /* - * spacer, - */ minusButton, plusButton); + header = new HBox(getDescrLabel(), getCountLabel(), hashIV, tagIV, minusButton, plusButton); header.setMinWidth(USE_PREF_SIZE); header.setPadding(new Insets(2, 5, 2, 5)); @@ -244,12 +274,17 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A show(plusButton, showControls); } + /** + * make a new filter intersecting the global filter with description and + * type filters to restrict sub-clusters + * + */ RootFilter getSubClusterFilter() { - RootFilter combinedFilter = eventsModel.filterProperty().get().copyOf(); - //make a new filter intersecting the global filter with description and type filters to restrict sub-clusters - combinedFilter.getSubFilters().addAll(new DescriptionFilter(getEventBundle().getDescriptionLOD(), getDescription()), + RootFilter subClusterFilter = eventsModel.filterProperty().get().copyOf(); + subClusterFilter.getSubFilters().addAll( + new DescriptionFilter(getEventBundle().getDescriptionLOD(), getDescription()), new TypeFilter(getEventType())); - return combinedFilter; + return subClusterFilter; } abstract Collection makeBundlesFromClusters(List eventClusters); @@ -324,59 +359,84 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A } /** - * loads sub-clusters at the given Description LOD + * loads sub-bundles at the given Description LOD, continues * - * @param newDescriptionLOD + * @param requestedDescrLoD + * @param expand */ - final synchronized void loadSubClusters(DescriptionLOD newDescriptionLOD) { + private synchronized void loadSubBundles(DescriptionLOD.RelativeDetail relativeDetail) { subNodePane.getChildren().clear(); - if (newDescriptionLOD == getEventBundle().getDescriptionLOD()) { + if (descLOD.get().withRelativeDetail(relativeDetail) == getEventBundle().getDescriptionLOD()) { + descLOD.set(getEventBundle().getDescriptionLOD()); showSpans(true); - getChart().setRequiresLayout(true); - getChart().requestChartLayout(); + chart.setRequiresLayout(true); + chart.requestChartLayout(); } else { showSpans(false); - RootFilter combinedFilter = getSubClusterFilter(); - //make a new end inclusive span (to 'filter' with) - final Interval span = new Interval(getEventBundle().getStartMillis(), getEventBundle().getEndMillis() + 1000); + // make new ZoomParams to query with + final RootFilter subClusterFilter = getSubClusterFilter(); + /* + * We need to extend end time because for the query by one second, + * because it is treated as an open interval but we want to include + * events at exactly the time of the last event in this cluster + */ + final Interval subClusterSpan = new Interval(getEventBundle().getStartMillis(), getEventBundle().getEndMillis() + 1000); + final EventTypeZoomLevel eventTypeZoomLevel = eventsModel.eventTypeZoomProperty().get(); + final ZoomParams zoomParams = new ZoomParams(subClusterSpan, eventTypeZoomLevel, subClusterFilter, getDescLOD()); - //make a task to load the subnodes - LoggedTask> loggedTask = new LoggedTask>( + LoggedTask> loggedTask; + loggedTask = new LoggedTask>( NbBundle.getMessage(this.getClass(), "AggregateEventNode.loggedTask.name"), true) { + private Collection bundles; + private volatile DescriptionLOD loadedDescriptionLoD = getDescLOD().withRelativeDetail(relativeDetail); + private DescriptionLOD next = loadedDescriptionLoD; @Override protected List call() throws Exception { - //query for the sub-clusters - List aggregatedEvents = eventsModel.getAggregatedEvents(new ZoomParams(span, - eventsModel.eventTypeZoomProperty().get(), - combinedFilter, - newDescriptionLOD)); + do { + loadedDescriptionLoD = next; + if (loadedDescriptionLoD == getEventBundle().getDescriptionLOD()) { + return Collections.emptyList(); + } + bundles = loadBundles(); + next = loadedDescriptionLoD.withRelativeDetail(relativeDetail); + } while (bundles.size() == 1 && nonNull(next)); - return makeBundlesFromClusters(aggregatedEvents).stream() - .map(aggEvent -> { - return getNodeForCluser(aggEvent); - }).collect(Collectors.toList()); // return list of AggregateEventNodes representing subclusters + // return list of AbstractDetailViewNodes representing sub-bundles + return bundles.stream() + .map(AbstractDetailViewNode.this::getNodeForBundle) + .collect(Collectors.toList()); + } + + private Collection loadBundles() { + return makeBundlesFromClusters(eventsModel.getEventClusters(zoomParams.withDescrLOD(loadedDescriptionLoD))); } @Override protected void succeeded() { + chart.setCursor(Cursor.WAIT); try { - getChart().setCursor(Cursor.WAIT); + List subBundleNodes = get(); + if (subBundleNodes.isEmpty()) { + showSpans(true); + } else { + showSpans(false); + } + descLOD.set(loadedDescriptionLoD); //assign subNodes and request chart layout - subNodePane.getChildren().setAll(get()); - setDescriptionVisibility(descrVis); - getChart().setRequiresLayout(true); - getChart().requestChartLayout(); - getChart().setCursor(null); + subNodePane.getChildren().setAll(subBundleNodes); + chart.setRequiresLayout(true); + chart.requestChartLayout(); } catch (InterruptedException | ExecutionException ex) { LOGGER.log(Level.SEVERE, "Error loading subnodes", ex); } + chart.setCursor(null); } }; //start task - getChart().getController().monitorTask(loggedTask); + chart.getController().monitorTask(loggedTask); } } @@ -390,7 +450,7 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A this.descrVis = descrVis; final int size = getEventBundle().getEventIDs().size(); - switch (descrVis) { + switch (this.descrVis) { case COUNT_ONLY: descrLabel.setText(""); countLabel.setText(String.valueOf(size)); @@ -411,7 +471,7 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A } } - abstract S getNodeForCluser(T cluster); + abstract S getNodeForBundle(T bundle); /** * event handler used for mouse events on {@link AggregateEventNode}s @@ -432,10 +492,10 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A } else if (t.isShortcutDown()) { chart.selectedNodes.removeAll(AbstractDetailViewNode.this); } else if (t.getClickCount() > 1) { - final DescriptionLOD next = descLOD.get().next(); + final DescriptionLOD next = descLOD.get().moreDetailed(); if (next != null) { - loadSubClusters(next); - descLOD.set(next); + loadSubBundles(DescriptionLOD.RelativeDetail.MORE); + } } else { chart.selectedNodes.setAll(AbstractDetailViewNode.this); @@ -461,14 +521,15 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A private class ExpandClusterAction extends Action { - public ExpandClusterAction() { + ExpandClusterAction() { super("Expand"); + setGraphic(new ImageView(PLUS)); setEventHandler((ActionEvent t) -> { - final DescriptionLOD next = descLOD.get().next(); + final DescriptionLOD next = descLOD.get().moreDetailed(); if (next != null) { - loadSubClusters(next); - descLOD.set(next); + loadSubBundles(DescriptionLOD.RelativeDetail.MORE); + } }); disabledProperty().bind(descLOD.isEqualTo(DescriptionLOD.FULL)); @@ -477,15 +538,14 @@ public abstract class AbstractDetailViewNode< T extends EventBundle, S extends A private class CollapseClusterAction extends Action { - public CollapseClusterAction() { + CollapseClusterAction() { super("Collapse"); setGraphic(new ImageView(MINUS)); setEventHandler((ActionEvent t) -> { - final DescriptionLOD previous = descLOD.get().previous(); + final DescriptionLOD previous = descLOD.get().lessDetailed(); if (previous != null) { - loadSubClusters(previous); - descLOD.set(previous); + loadSubBundles(DescriptionLOD.RelativeDetail.LESS); } }); disabledProperty().bind(descLOD.isEqualTo(getEventBundle().getDescriptionLOD())); diff --git a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java index b07adac60b..7a3dfc3ac3 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/ui/detailview/EventClusterNode.java @@ -66,8 +66,8 @@ public class EventClusterNode extends AbstractDetailViewNode(); try { - for (TimeLineEvent tle : eventsModel.getEventsById(getEventCluster().getEventIDsWithHashHits())) { - Set hashSetNames = sleuthkitCase.getAbstractFileById(tle.getFileID()).getHashSetNames(); + for (TimeLineEvent tle : getEventsModel().getEventsById(getEventCluster().getEventIDsWithHashHits())) { + Set hashSetNames = getSleuthkitCase().getAbstractFileById(tle.getFileID()).getHashSetNames(); for (String hashSetName : hashSetNames) { hashSetCounts.merge(hashSetName, 1L, Long::sum); } @@ -99,7 +99,7 @@ public class EventClusterNode extends AbstractDetailViewNode tagCounts = new HashMap<>(); if (!getEventCluster().getEventIDsWithTags().isEmpty()) { - tagCounts.putAll(eventsModel.getTagCountsByTagName(getEventCluster().getEventIDsWithTags())); + tagCounts.putAll(getEventsModel().getTagCountsByTagName(getEventCluster().getEventIDsWithTags())); } @@ -174,7 +174,7 @@ public class EventClusterNode extends AbstractDetailViewNode { - private static final Logger LOGGER = Logger.getLogger(EventClusterNode.class.getName()); - private final HBox rangesHBox = new HBox(); EventStripeNode(EventStripe eventStripe, EventStripeNode parentNode, EventDetailChart chart) { super(chart, eventStripe, parentNode); minWidthProperty().bind(rangesHBox.widthProperty()); - final VBox internalVBox = new VBox(header, subNodePane); + final VBox internalVBox = new VBox(header, getSubNodePane()); internalVBox.setAlignment(Pos.CENTER_LEFT); for (Range range : eventStripe.getRanges()) { @@ -57,7 +54,7 @@ public class EventStripeNode extends AbstractDetailViewNode