preliminary work to sort groups by uncategorized file count

Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java

@@ -294,7 +294,8 @@ public class ImageUtils {
                         || (conditionalMimes.contains(mimeType.toLowerCase()) && supportedExtension.contains(extension));
             }
         } catch (FileTypeDetector.FileTypeDetectorInitException | TskCoreException ex) {
-            LOGGER.log(Level.WARNING, "Failed to look up mimetype for " + getContentPathSafe(file) + " using FileTypeDetector.  Fallingback on AbstractFile.isMimeType", ex); //NOI18N
+            LOGGER.log(Level.WARNING, "Failed to look up mimetype for {0} using FileTypeDetector:{1}", new Object[]{getContentPathSafe(file), ex.toString()}); //NOI18N
+            LOGGER.log(Level.INFO, "Falling back on AbstractFile.isMimeType"); //NOI18N
             AbstractFile.MimeMatchEnum mimeMatch = file.isMimeType(supportedMimeTypes);
             if (mimeMatch == AbstractFile.MimeMatchEnum.TRUE) {
                 return true;
@@ -748,6 +749,7 @@ public class ImageUtils {
             }
 
             return SwingFXUtils.toFXImage(thumbnail, null);
+
         }
 
         /**

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java

@@ -23,18 +23,23 @@ import com.google.common.cache.LoadingCache;
 import com.google.common.eventbus.AsyncEventBus;
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.LongAdder;
 import java.util.logging.Level;
 import javax.annotation.concurrent.Immutable;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
-import org.sleuthkit.autopsy.coreutils.Logger;
+import org.openide.util.Exceptions;
 import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent;
 import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent;
+import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.imagegallery.ImageGalleryController;
 import org.sleuthkit.datamodel.ContentTag;
+import org.sleuthkit.datamodel.SleuthkitCase;
 import org.sleuthkit.datamodel.TagName;
 import org.sleuthkit.datamodel.TskCoreException;
 
@@ -77,14 +82,14 @@ public class CategoryManager {
      * the count related methods go through this cache, which loads initial
      * values from the database if needed.
      */
-    private final LoadingCache<Category, LongAdder> categoryCounts
+    private final LoadingCache<Category, LongAdder> categoryCounts =
-            = CacheBuilder.newBuilder().build(CacheLoader.from(this::getCategoryCountHelper));
+            CacheBuilder.newBuilder().build(CacheLoader.from(this::getCategoryCountHelper));
     /**
      * cached TagNames corresponding to Categories, looked up from
      * autopsyTagManager at initial request or if invalidated by case change.
      */
-    private final LoadingCache<Category, TagName> catTagNameMap = CacheBuilder.newBuilder().build(CacheLoader.from(cat
+    private final LoadingCache<Category, TagName> catTagNameMap = CacheBuilder.newBuilder().build(CacheLoader.from(cat ->
-            -> getController().getTagsManager().getTagName(cat)));
+            getController().getTagsManager().getTagName(cat)));
 
     public CategoryManager(ImageGalleryController controller) {
         this.controller = controller;
@@ -115,6 +120,32 @@ public class CategoryManager {
         fireChange(Collections.emptyList(), null);
     }
 
+    public long getCategoryCount(Category cat, Collection<Long> fileIDs) {
+        String name = "select count(obj_id) from tsk_files "
+                + " where "
+                + "         obj_id in (" + StringUtils.join(fileIDs, ",") + " ) "
+                + "     and obj_id not in ( "
+                + "         select obj_id from  content_tags where "
+                + "             content_tags.tag_name_id in ("
+                + getTagName(Category.FIVE).getId() + " ,"
+                + getTagName(Category.FOUR).getId() + " ,"
+                + getTagName(Category.THREE).getId() + " ,"
+                + getTagName(Category.TWO).getId() + " ,"
+                + getTagName(Category.ONE).getId()
+                + " ))";
+        try (SleuthkitCase.CaseDbQuery executeQuery = controller.getSleuthKitCase().executeQuery(name);
+                ResultSet resultSet = executeQuery.getResultSet();) {
+            while (resultSet.next()) {
+                return resultSet.getLong("count(obj_id)");
+            }
+        } catch (SQLException sQLException) {
+        } catch (TskCoreException ex) {
+            Exceptions.printStackTrace(ex);
+
+        }
+        return -1;
+    }
+
     /**
      * get the number of file with the given {@link Category}
      *
@@ -194,7 +225,7 @@ public class CategoryManager {
      *
      * @param listner
      */
-    public void registerListener(Object listner) {
+    synchronized public void registerListener(Object listner) {
         categoryEventBus.register(listner);
     }
 
@@ -203,7 +234,7 @@ public class CategoryManager {
      *
      * @param listener
      */
-    public void unregisterListener(Object listener) {
+    synchronized public void unregisterListener(Object listener) {
 
         try {
             categoryEventBus.unregister(listener);
@@ -284,8 +315,7 @@ public class CategoryManager {
 
     /**
      * Event broadcast to various UI componenets when one or more files'
-     * category
+     * category has been changed
-     * has been changed
      */
     @Immutable
     public static class CategoryChangeEvent {

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java

@@ -41,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 import javax.annotation.concurrent.GuardedBy;
 import javax.swing.SortOrder;
 import org.apache.commons.lang3.StringUtils;
@@ -1257,6 +1258,38 @@ public final class DrawableDB {
         }
         return -1;
     }
+    /**
+     * get the id s of files with the given category.
+     *
+     * NOTE: although the category data is stored in autopsy as Tags, this
+     * method is provided on DrawableDb to provide a single point of access for
+     * ImageGallery data.
+     *
+     * //TODO: think about moving this and similar methods that don't actually
+     * get their data form the drawabledb to a layer wrapping the drawable db:
+     * something like ImageGalleryCaseData?
+     *
+     * @param cat the category to count the number of files for
+     *
+     * @return the number of the with the given category
+     */
+    public Set<Long> getCategoryFileIds(Category cat) {
+        try {
+            TagName tagName = controller.getTagsManager().getTagName(cat);
+            if (nonNull(tagName)) {
+                return tskCase.getContentTagsByTagName(tagName).stream()
+                        .map(ContentTag::getContent)
+                        .map(Content::getId)
+                        .filter(this::isInDB)
+                        .collect(Collectors.toSet());
+            }
+        } catch (IllegalStateException ex) {
+            LOGGER.log(Level.WARNING, "Case closed while getting files");
+        } catch (TskCoreException ex1) {
+            LOGGER.log(Level.SEVERE, "Failed to get content tags by tag name.", ex1);
+        }
+        return Collections.emptySet();
+    }
 
     /**
      * inner class that can reference access database connection

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java

@@ -18,14 +18,20 @@
  */
 package org.sleuthkit.autopsy.imagegallery.datamodel.grouping;
 
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.Subscribe;
 import java.util.Objects;
 import java.util.Set;
 import java.util.logging.Level;
 import javafx.beans.property.ReadOnlyBooleanWrapper;
+import javafx.beans.property.ReadOnlyLongProperty;
+import javafx.beans.property.ReadOnlyLongWrapper;
 import javafx.collections.FXCollections;
 import javafx.collections.ObservableList;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.imagegallery.ImageGalleryController;
+import org.sleuthkit.autopsy.imagegallery.datamodel.Category;
+import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager;
 import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute;
 
 /**
@@ -45,7 +51,9 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
 
     //cache the number of files in this groups with hashset hits
     private long hashSetHitsCount = -1;
+
     private final ReadOnlyBooleanWrapper seen = new ReadOnlyBooleanWrapper(false);
+    private final ReadOnlyLongWrapper uncatCount = new ReadOnlyLongWrapper(-1);
 
     @SuppressWarnings("ReturnOfCollectionOrArrayField")
     synchronized public ObservableList<Long> fileIds() {
@@ -74,6 +82,7 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
         this.groupKey = groupKey;
         this.fileIDs.setAll(filesInGroup);
         this.seen.set(seen);
+        getUncategorizedCount();
     }
 
     synchronized public int getSize() {
@@ -92,6 +101,10 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
         hashSetHitsCount = -1;
     }
 
+    synchronized private void invalidateUncatCount() {
+        uncatCount.set(-1);
+    }
+
     /**
      * @return the number of files in this group that have hash set hits
      */
@@ -110,6 +123,24 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
         return hashSetHitsCount;
     }
 
+    final synchronized public long getUncategorizedCount() {
+        if (uncatCount.get() < 0) {
+            try {
+                uncatCount.set(ImageGalleryController.getDefault().getCategoryManager().getCategoryCount(Category.ZERO, fileIDs));
+
+            } catch (IllegalStateException | NullPointerException ex) {
+                LOGGER.log(Level.WARNING, "could not access case during getFilesWithHashSetHitsCount()");
+            }
+        }
+
+        return uncatCount.get();
+    }
+
+    public ReadOnlyLongProperty uncatCountProperty() {
+        return uncatCount.getReadOnlyProperty();
+    }
+
+
     @Override
     public String toString() {
         return "Grouping{ keyProp=" + groupKey + '}';
@@ -136,6 +167,7 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
 
     synchronized void addFile(Long f) {
         invalidateHashSetHitsCount();
+        invalidateUncatCount();
         if (fileIDs.contains(f) == false) {
             fileIDs.add(f);
             seen.set(false);
@@ -144,6 +176,7 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
 
     synchronized void setFiles(Set<? extends Long> newFileIds) {
         invalidateHashSetHitsCount();
+        invalidateUncatCount();
         boolean filesRemoved = fileIDs.removeIf((Long t) -> newFileIds.contains(t) == false);
         if (filesRemoved) {
             seen.set(false);
@@ -158,6 +191,7 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
 
     synchronized void removeFile(Long f) {
         invalidateHashSetHitsCount();
+        invalidateUncatCount();
         if (fileIDs.removeAll(f)) {
             seen.set(false);
         }
@@ -181,4 +215,10 @@ public class DrawableGroup implements Comparable<DrawableGroup> {
         return seen.get();
     }
 
+    @Subscribe
+    synchronized public void handleCatChange(CategoryManager.CategoryChangeEvent event) {
+        if (Iterables.any(event.getFileIDs(), fileIDs::contains)) {
+            invalidateUncatCount();
+        }
+    }
 }

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java

@@ -216,10 +216,14 @@ public class GroupManager {
         groupBy = DrawableAttribute.PATH;
         sortOrder = SortOrder.ASCENDING;
         Platform.runLater(() -> {
+            unSeenGroups.forEach(controller.getCategoryManager()::unregisterListener);
             unSeenGroups.clear();
+            analyzedGroups.forEach(controller.getCategoryManager()::unregisterListener);
             analyzedGroups.clear();
+
         });
         synchronized (groupMap) {
+            groupMap.values().forEach(controller.getCategoryManager()::unregisterListener);
             groupMap.clear();
         }
         db = null;
@@ -611,9 +615,11 @@ public class GroupManager {
                         synchronized (groupMap) {
                             if (groupMap.containsKey(groupKey)) {
                                 group = groupMap.get(groupKey);
+
                                 group.setFiles(ObjectUtils.defaultIfNull(fileIDs, Collections.emptySet()));
                             } else {
                                 group = new DrawableGroup(groupKey, fileIDs, groupSeen);
+                                controller.getCategoryManager().registerListener(group);
                                 group.seenProperty().addListener((o, oldSeen, newSeen) -> {
                                     markGroupSeen(group, newSeen);
                                 });

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupSortBy.java

@@ -49,11 +49,7 @@ public enum GroupSortBy implements ComparatorProvider {
 
                 @Override
                 public <A extends Comparable<A>> Comparator<A> getValueComparator(final DrawableAttribute<A> attr, final SortOrder sortOrder) {
-                    return (A v1, A v2) -> {
+                    return getDefaultValueComparator(attr, sortOrder);
-                        DrawableGroup g1 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<A>(attr, v1));
-                        DrawableGroup g2 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<A>(attr, v2));
-                        return getGrpComparator(sortOrder).compare(g1, g2);
-                    };
                 }
             },
     /**
@@ -101,12 +97,7 @@ public enum GroupSortBy implements ComparatorProvider {
 
                 @Override
                 public <A extends Comparable<A>> Comparator<A> getValueComparator(DrawableAttribute<A> attr, SortOrder sortOrder) {
-                    return (A v1, A v2) -> {
+                    return getDefaultValueComparator(attr, sortOrder);
-                        DrawableGroup g1 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<A>(attr, v1));
-                        DrawableGroup g2 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<A>(attr, v2));
-
-                        return getGrpComparator(sortOrder).compare(g1, g2);
-                    };
                 }
             };
 
@@ -151,12 +142,12 @@ public enum GroupSortBy implements ComparatorProvider {
         return sortOrderEnabled;
     }
 
-    private static <T> Comparator<T> applySortOrder(final SortOrder sortOrder, Comparator<T> comparingInt) {
+    private static <T> Comparator<T> applySortOrder(final SortOrder sortOrder, Comparator<T> comparator) {
         switch (sortOrder) {
             case ASCENDING:
-                return comparingInt;
+                return comparator;
             case DESCENDING:
-                return comparingInt.reversed();
+                return comparator.reversed();
             case UNSORTED:
             default:
                 return new NoOpComparator<>();
@@ -170,11 +161,12 @@ public enum GroupSortBy implements ComparatorProvider {
             return 0;
         }
     }
+
 }
 
 /**
  * * implementers of this interface must provide a method to compare
- * ({@link  Comparable}) values and Groupings based on an
+ * ({@link Comparable}) values and Groupings based on an
  * {@link DrawableAttribute} and a {@link SortOrder}
  */
 interface ComparatorProvider {
@@ -182,4 +174,13 @@ interface ComparatorProvider {
     <A extends Comparable<A>> Comparator<A> getValueComparator(DrawableAttribute<A> attr, SortOrder sortOrder);
 
     Comparator<DrawableGroup> getGrpComparator(SortOrder sortOrder);
+
+    default <A extends Comparable<A>> Comparator<A> getDefaultValueComparator(DrawableAttribute<A> attr, SortOrder sortOrder) {
+        return (A v1, A v2) -> {
+            DrawableGroup g1 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<>(attr, v1));
+            DrawableGroup g2 = ImageGalleryController.getDefault().getGroupManager().getGroupForKey(new GroupKey<>(attr, v2));
+
+            return getGrpComparator(sortOrder).compare(g1, g2);
+        };
+    }
 }

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupTreeCell.java

@@ -94,7 +94,9 @@ class GroupTreeCell extends TreeCell<TreeNode> {
                 .map(TreeNode::getGroup)
                 .ifPresent(group -> {
                     group.fileIds().removeListener(fileCountListener);
-                    group.seenProperty().removeListener(seenListener);
+            group.seenProperty().removeListener(seenListener);
+            group.uncatCountProperty().removeListener(fileCountListener);
+
                 });
 
         super.updateItem(treeNode, empty);
@@ -124,6 +126,9 @@ class GroupTreeCell extends TreeCell<TreeNode> {
                 //if the seen state of this group changes update its style
                 treeNode.getGroup().seenProperty().addListener(seenListener);
 
+                treeNode.getGroup().uncatCountProperty().addListener(fileCountListener);
+
+
                 //and use icon corresponding to group type
                 final Image icon = treeNode.getGroup().groupKey.getAttribute().getIcon();
                 final String text = getGroupName() + getCountsText();
@@ -170,8 +175,8 @@ class GroupTreeCell extends TreeCell<TreeNode> {
                 .map(TreeNode::getGroup)
                 .map(group -> " ("
                         + ((group.getGroupByAttribute() == DrawableAttribute.HASHSET)
-                                ? Integer.toString(group.getSize())
+                                ? group.getSize() + "/" + group.getUncategorizedCount()
-                                : group.getHashSetHitsCount() + "/" + group.getSize())
+                                : group.getHashSetHitsCount() + "/" + group.getSize() + "/" + group.getUncategorizedCount())
                         + ")"
                 ).orElse(""); //if item is null or group is null
     }

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java

@@ -18,6 +18,7 @@
  */
 package org.sleuthkit.autopsy.imagegallery.gui.navpanel;
 
+import com.google.common.eventbus.Subscribe;
 import java.util.Arrays;
 import java.util.List;
 import javafx.application.Platform;
@@ -41,6 +42,7 @@ import org.sleuthkit.autopsy.coreutils.ThreadConfined;
 import org.sleuthkit.autopsy.coreutils.ThreadConfined.ThreadType;
 import org.sleuthkit.autopsy.imagegallery.FXMLConstructor;
 import org.sleuthkit.autopsy.imagegallery.ImageGalleryController;
+import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager;
 import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute;
 import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.DrawableGroup;
 import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState;
@@ -92,6 +94,11 @@ final public class NavPanel extends TabPane {
         FXMLConstructor.construct(this, "NavPanel.fxml");
     }
 
+    @Subscribe
+    public void handleCategoryChange(CategoryManager.CategoryChangeEvent event) {
+        resortHashTree();
+    }
+
     @FXML
     void initialize() {
         assert hashTab != null : "fx:id=\"hashTab\" was not injected: check your FXML file 'NavPanel.fxml'.";
@@ -108,6 +115,11 @@ final public class NavPanel extends TabPane {
         sortByBox.setItems(FXCollections.observableArrayList(FXCollections.observableArrayList(TreeNodeComparators.values())));
         sortByBox.getSelectionModel().select(TreeNodeComparators.HIT_COUNT);
         sortByBox.getSelectionModel().selectedItemProperty().addListener(o -> resortHashTree());
+            if (sortByBox.getSelectionModel().getSelectedItem() == TreeNodeComparators.UNCATEGORIZED_COUNT) {
+                controller.getCategoryManager().registerListener(NavPanel.this);
+            } else {
+                controller.getCategoryManager().unregisterListener(NavPanel.this);
+            }
 
         configureTree(navTree, navTreeRoot);
         configureTree(hashTree, hashTreeRoot);

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/TreeNodeComparators.java

@@ -27,6 +27,12 @@ import javafx.scene.control.TreeItem;
  */
 enum TreeNodeComparators implements Comparator<TreeItem<TreeNode>>, NonNullCompareable {
 
+    UNCATEGORIZED_COUNT("Uncategorized Count") {
+                @Override
+                public int nonNullCompare(TreeItem<TreeNode> o1, TreeItem<TreeNode> o2) {
+                    return -Long.compare(o1.getValue().getGroup().getUncategorizedCount(), o2.getValue().getGroup().getUncategorizedCount());
+                }
+            },
     ALPHABETICAL("Group Name") {
                 @Override
                 public int nonNullCompare(TreeItem<TreeNode> o1, TreeItem<TreeNode> o2) {