From fc94e2b6391b36944c1cbe0a33c093f34a0b46e7 Mon Sep 17 00:00:00 2001 From: jmillman Date: Fri, 8 Jan 2016 16:07:36 -0500 Subject: [PATCH] preliminary work to sort groups by uncategorized file count --- .../autopsy/coreutils/ImageUtils.java | 4 +- .../datamodel/CategoryManager.java | 48 +++++++++++++++---- .../imagegallery/datamodel/DrawableDB.java | 33 +++++++++++++ .../datamodel/grouping/DrawableGroup.java | 40 ++++++++++++++++ .../datamodel/grouping/GroupManager.java | 6 +++ .../datamodel/grouping/GroupSortBy.java | 31 ++++++------ .../gui/navpanel/GroupTreeCell.java | 11 +++-- .../imagegallery/gui/navpanel/NavPanel.java | 12 +++++ .../gui/navpanel/TreeNodeComparators.java | 6 +++ 9 files changed, 163 insertions(+), 28 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java index db5d1bc9d9..af82cec91b 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java @@ -294,7 +294,8 @@ public class ImageUtils { || (conditionalMimes.contains(mimeType.toLowerCase()) && supportedExtension.contains(extension)); } } catch (FileTypeDetector.FileTypeDetectorInitException | TskCoreException ex) { - LOGGER.log(Level.WARNING, "Failed to look up mimetype for " + getContentPathSafe(file) + " using FileTypeDetector. Fallingback on AbstractFile.isMimeType", ex); //NOI18N + LOGGER.log(Level.WARNING, "Failed to look up mimetype for {0} using FileTypeDetector:{1}", new Object[]{getContentPathSafe(file), ex.toString()}); //NOI18N + LOGGER.log(Level.INFO, "Falling back on AbstractFile.isMimeType"); //NOI18N AbstractFile.MimeMatchEnum mimeMatch = file.isMimeType(supportedMimeTypes); if (mimeMatch == AbstractFile.MimeMatchEnum.TRUE) { return true; @@ -748,6 +749,7 @@ public class ImageUtils { } return SwingFXUtils.toFXImage(thumbnail, null); + } /** diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java index ab92d6d862..9a69a2b9d0 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java @@ -23,18 +23,23 @@ import com.google.common.cache.LoadingCache; import com.google.common.eventbus.AsyncEventBus; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Collection; import java.util.Collections; import java.util.concurrent.Executors; import java.util.concurrent.atomic.LongAdder; import java.util.logging.Level; import javax.annotation.concurrent.Immutable; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; -import org.sleuthkit.autopsy.coreutils.Logger; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.imagegallery.ImageGalleryController; import org.sleuthkit.datamodel.ContentTag; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; @@ -77,14 +82,14 @@ public class CategoryManager { * the count related methods go through this cache, which loads initial * values from the database if needed. */ - private final LoadingCache categoryCounts - = CacheBuilder.newBuilder().build(CacheLoader.from(this::getCategoryCountHelper)); + private final LoadingCache categoryCounts = + CacheBuilder.newBuilder().build(CacheLoader.from(this::getCategoryCountHelper)); /** * cached TagNames corresponding to Categories, looked up from * autopsyTagManager at initial request or if invalidated by case change. */ - private final LoadingCache catTagNameMap = CacheBuilder.newBuilder().build(CacheLoader.from(cat - -> getController().getTagsManager().getTagName(cat))); + private final LoadingCache catTagNameMap = CacheBuilder.newBuilder().build(CacheLoader.from(cat -> + getController().getTagsManager().getTagName(cat))); public CategoryManager(ImageGalleryController controller) { this.controller = controller; @@ -115,6 +120,32 @@ public class CategoryManager { fireChange(Collections.emptyList(), null); } + public long getCategoryCount(Category cat, Collection fileIDs) { + String name = "select count(obj_id) from tsk_files " + + " where " + + " obj_id in (" + StringUtils.join(fileIDs, ",") + " ) " + + " and obj_id not in ( " + + " select obj_id from content_tags where " + + " content_tags.tag_name_id in (" + + getTagName(Category.FIVE).getId() + " ," + + getTagName(Category.FOUR).getId() + " ," + + getTagName(Category.THREE).getId() + " ," + + getTagName(Category.TWO).getId() + " ," + + getTagName(Category.ONE).getId() + + " ))"; + try (SleuthkitCase.CaseDbQuery executeQuery = controller.getSleuthKitCase().executeQuery(name); + ResultSet resultSet = executeQuery.getResultSet();) { + while (resultSet.next()) { + return resultSet.getLong("count(obj_id)"); + } + } catch (SQLException sQLException) { + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); + + } + return -1; + } + /** * get the number of file with the given {@link Category} * @@ -194,7 +225,7 @@ public class CategoryManager { * * @param listner */ - public void registerListener(Object listner) { + synchronized public void registerListener(Object listner) { categoryEventBus.register(listner); } @@ -203,7 +234,7 @@ public class CategoryManager { * * @param listener */ - public void unregisterListener(Object listener) { + synchronized public void unregisterListener(Object listener) { try { categoryEventBus.unregister(listener); @@ -284,8 +315,7 @@ public class CategoryManager { /** * Event broadcast to various UI componenets when one or more files' - * category - * has been changed + * category has been changed */ @Immutable public static class CategoryChangeEvent { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 9aa9a059d4..506a8e5ae8 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -41,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; +import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; import javax.swing.SortOrder; import org.apache.commons.lang3.StringUtils; @@ -1257,6 +1258,38 @@ public final class DrawableDB { } return -1; } + /** + * get the id s of files with the given category. + * + * NOTE: although the category data is stored in autopsy as Tags, this + * method is provided on DrawableDb to provide a single point of access for + * ImageGallery data. + * + * //TODO: think about moving this and similar methods that don't actually + * get their data form the drawabledb to a layer wrapping the drawable db: + * something like ImageGalleryCaseData? + * + * @param cat the category to count the number of files for + * + * @return the number of the with the given category + */ + public Set getCategoryFileIds(Category cat) { + try { + TagName tagName = controller.getTagsManager().getTagName(cat); + if (nonNull(tagName)) { + return tskCase.getContentTagsByTagName(tagName).stream() + .map(ContentTag::getContent) + .map(Content::getId) + .filter(this::isInDB) + .collect(Collectors.toSet()); + } + } catch (IllegalStateException ex) { + LOGGER.log(Level.WARNING, "Case closed while getting files"); + } catch (TskCoreException ex1) { + LOGGER.log(Level.SEVERE, "Failed to get content tags by tag name.", ex1); + } + return Collections.emptySet(); + } /** * inner class that can reference access database connection diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java index ac0493d75e..6357263574 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java @@ -18,14 +18,20 @@ */ package org.sleuthkit.autopsy.imagegallery.datamodel.grouping; +import com.google.common.collect.Iterables; +import com.google.common.eventbus.Subscribe; import java.util.Objects; import java.util.Set; import java.util.logging.Level; import javafx.beans.property.ReadOnlyBooleanWrapper; +import javafx.beans.property.ReadOnlyLongProperty; +import javafx.beans.property.ReadOnlyLongWrapper; import javafx.collections.FXCollections; import javafx.collections.ObservableList; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.imagegallery.ImageGalleryController; +import org.sleuthkit.autopsy.imagegallery.datamodel.Category; +import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute; /** @@ -45,7 +51,9 @@ public class DrawableGroup implements Comparable { //cache the number of files in this groups with hashset hits private long hashSetHitsCount = -1; + private final ReadOnlyBooleanWrapper seen = new ReadOnlyBooleanWrapper(false); + private final ReadOnlyLongWrapper uncatCount = new ReadOnlyLongWrapper(-1); @SuppressWarnings("ReturnOfCollectionOrArrayField") synchronized public ObservableList fileIds() { @@ -74,6 +82,7 @@ public class DrawableGroup implements Comparable { this.groupKey = groupKey; this.fileIDs.setAll(filesInGroup); this.seen.set(seen); + getUncategorizedCount(); } synchronized public int getSize() { @@ -92,6 +101,10 @@ public class DrawableGroup implements Comparable { hashSetHitsCount = -1; } + synchronized private void invalidateUncatCount() { + uncatCount.set(-1); + } + /** * @return the number of files in this group that have hash set hits */ @@ -110,6 +123,24 @@ public class DrawableGroup implements Comparable { return hashSetHitsCount; } + final synchronized public long getUncategorizedCount() { + if (uncatCount.get() < 0) { + try { + uncatCount.set(ImageGalleryController.getDefault().getCategoryManager().getCategoryCount(Category.ZERO, fileIDs)); + + } catch (IllegalStateException | NullPointerException ex) { + LOGGER.log(Level.WARNING, "could not access case during getFilesWithHashSetHitsCount()"); + } + } + + return uncatCount.get(); + } + + public ReadOnlyLongProperty uncatCountProperty() { + return uncatCount.getReadOnlyProperty(); + } + + @Override public String toString() { return "Grouping{ keyProp=" + groupKey + '}'; @@ -136,6 +167,7 @@ public class DrawableGroup implements Comparable { synchronized void addFile(Long f) { invalidateHashSetHitsCount(); + invalidateUncatCount(); if (fileIDs.contains(f) == false) { fileIDs.add(f); seen.set(false); @@ -144,6 +176,7 @@ public class DrawableGroup implements Comparable { synchronized void setFiles(Set newFileIds) { invalidateHashSetHitsCount(); + invalidateUncatCount(); boolean filesRemoved = fileIDs.removeIf((Long t) -> newFileIds.contains(t) == false); if (filesRemoved) { seen.set(false); @@ -158,6 +191,7 @@ public class DrawableGroup implements Comparable { synchronized void removeFile(Long f) { invalidateHashSetHitsCount(); + invalidateUncatCount(); if (fileIDs.removeAll(f)) { seen.set(false); } @@ -181,4 +215,10 @@ public class DrawableGroup implements Comparable { return seen.get(); } + @Subscribe + synchronized public void handleCatChange(CategoryManager.CategoryChangeEvent event) { + if (Iterables.any(event.getFileIDs(), fileIDs::contains)) { + invalidateUncatCount(); + } + } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index dda6641097..0d0b02bfbc 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -216,10 +216,14 @@ public class GroupManager { groupBy = DrawableAttribute.PATH; sortOrder = SortOrder.ASCENDING; Platform.runLater(() -> { + unSeenGroups.forEach(controller.getCategoryManager()::unregisterListener); unSeenGroups.clear(); + analyzedGroups.forEach(controller.getCategoryManager()::unregisterListener); analyzedGroups.clear(); + }); synchronized (groupMap) { + groupMap.values().forEach(controller.getCategoryManager()::unregisterListener); groupMap.clear(); } db = null; @@ -611,9 +615,11 @@ public class GroupManager { synchronized (groupMap) { if (groupMap.containsKey(groupKey)) { group = groupMap.get(groupKey); + group.setFiles(ObjectUtils.defaultIfNull(fileIDs, Collections.emptySet())); } else { group = new DrawableGroup(groupKey, fileIDs, groupSeen); + controller.getCategoryManager().registerListener(group); group.seenProperty().addListener((o, oldSeen, newSeen) -> { markGroupSeen(group, newSeen); }); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupSortBy.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupSortBy.java index a7bdbb342e..de2c806105 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupSortBy.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupSortBy.java @@ -49,11 +49,7 @@ public enum GroupSortBy implements ComparatorProvider { @Override public > Comparator getValueComparator(final DrawableAttribute attr, final SortOrder sortOrder) { - return (A v1, A v2) -> { - DrawableGroup g1 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey(attr, v1)); - DrawableGroup g2 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey(attr, v2)); - return getGrpComparator(sortOrder).compare(g1, g2); - }; + return getDefaultValueComparator(attr, sortOrder); } }, /** @@ -101,12 +97,7 @@ public enum GroupSortBy implements ComparatorProvider { @Override public > Comparator getValueComparator(DrawableAttribute attr, SortOrder sortOrder) { - return (A v1, A v2) -> { - DrawableGroup g1 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey(attr, v1)); - DrawableGroup g2 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey(attr, v2)); - - return getGrpComparator(sortOrder).compare(g1, g2); - }; + return getDefaultValueComparator(attr, sortOrder); } }; @@ -151,12 +142,12 @@ public enum GroupSortBy implements ComparatorProvider { return sortOrderEnabled; } - private static Comparator applySortOrder(final SortOrder sortOrder, Comparator comparingInt) { + private static Comparator applySortOrder(final SortOrder sortOrder, Comparator comparator) { switch (sortOrder) { case ASCENDING: - return comparingInt; + return comparator; case DESCENDING: - return comparingInt.reversed(); + return comparator.reversed(); case UNSORTED: default: return new NoOpComparator<>(); @@ -170,11 +161,12 @@ public enum GroupSortBy implements ComparatorProvider { return 0; } } + } /** * * implementers of this interface must provide a method to compare - * ({@link Comparable}) values and Groupings based on an + * ({@link Comparable}) values and Groupings based on an * {@link DrawableAttribute} and a {@link SortOrder} */ interface ComparatorProvider { @@ -182,4 +174,13 @@ interface ComparatorProvider { > Comparator getValueComparator(DrawableAttribute attr, SortOrder sortOrder); Comparator getGrpComparator(SortOrder sortOrder); + + default > Comparator getDefaultValueComparator(DrawableAttribute attr, SortOrder sortOrder) { + return (A v1, A v2) -> { + DrawableGroup g1 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<>(attr, v1)); + DrawableGroup g2 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<>(attr, v2)); + + return getGrpComparator(sortOrder).compare(g1, g2); + }; + } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupTreeCell.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupTreeCell.java index 8618634aab..ebaa85098d 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupTreeCell.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupTreeCell.java @@ -94,7 +94,9 @@ class GroupTreeCell extends TreeCell { .map(TreeNode::getGroup) .ifPresent(group -> { group.fileIds().removeListener(fileCountListener); - group.seenProperty().removeListener(seenListener); + group.seenProperty().removeListener(seenListener); + group.uncatCountProperty().removeListener(fileCountListener); + }); super.updateItem(treeNode, empty); @@ -124,6 +126,9 @@ class GroupTreeCell extends TreeCell { //if the seen state of this group changes update its style treeNode.getGroup().seenProperty().addListener(seenListener); + treeNode.getGroup().uncatCountProperty().addListener(fileCountListener); + + //and use icon corresponding to group type final Image icon = treeNode.getGroup().groupKey.getAttribute().getIcon(); final String text = getGroupName() + getCountsText(); @@ -170,8 +175,8 @@ class GroupTreeCell extends TreeCell { .map(TreeNode::getGroup) .map(group -> " (" + ((group.getGroupByAttribute() == DrawableAttribute.HASHSET) - ? Integer.toString(group.getSize()) - : group.getHashSetHitsCount() + "/" + group.getSize()) + ? group.getSize() + "/" + group.getUncategorizedCount() + : group.getHashSetHitsCount() + "/" + group.getSize() + "/" + group.getUncategorizedCount()) + ")" ).orElse(""); //if item is null or group is null } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java index 84c28832e2..860f30e823 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.imagegallery.gui.navpanel; +import com.google.common.eventbus.Subscribe; import java.util.Arrays; import java.util.List; import javafx.application.Platform; @@ -41,6 +42,7 @@ import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.coreutils.ThreadConfined.ThreadType; import org.sleuthkit.autopsy.imagegallery.FXMLConstructor; import org.sleuthkit.autopsy.imagegallery.ImageGalleryController; +import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.DrawableGroup; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; @@ -92,6 +94,11 @@ final public class NavPanel extends TabPane { FXMLConstructor.construct(this, "NavPanel.fxml"); } + @Subscribe + public void handleCategoryChange(CategoryManager.CategoryChangeEvent event) { + resortHashTree(); + } + @FXML void initialize() { assert hashTab != null : "fx:id=\"hashTab\" was not injected: check your FXML file 'NavPanel.fxml'."; @@ -108,6 +115,11 @@ final public class NavPanel extends TabPane { sortByBox.setItems(FXCollections.observableArrayList(FXCollections.observableArrayList(TreeNodeComparators.values()))); sortByBox.getSelectionModel().select(TreeNodeComparators.HIT_COUNT); sortByBox.getSelectionModel().selectedItemProperty().addListener(o -> resortHashTree()); + if (sortByBox.getSelectionModel().getSelectedItem() == TreeNodeComparators.UNCATEGORIZED_COUNT) { + controller.getCategoryManager().registerListener(NavPanel.this); + } else { + controller.getCategoryManager().unregisterListener(NavPanel.this); + } configureTree(navTree, navTreeRoot); configureTree(hashTree, hashTreeRoot); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/TreeNodeComparators.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/TreeNodeComparators.java index 096dd681a1..23228ae0a3 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/TreeNodeComparators.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/TreeNodeComparators.java @@ -27,6 +27,12 @@ import javafx.scene.control.TreeItem; */ enum TreeNodeComparators implements Comparator>, NonNullCompareable { + UNCATEGORIZED_COUNT("Uncategorized Count") { + @Override + public int nonNullCompare(TreeItem o1, TreeItem o2) { + return -Long.compare(o1.getValue().getGroup().getUncategorizedCount(), o2.getValue().getGroup().getUncategorizedCount()); + } + }, ALPHABETICAL("Group Name") { @Override public int nonNullCompare(TreeItem o1, TreeItem o2) {