Merge pull request #1227 from APriestman/missingDirs

Various Image Gallery performance improvements.

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java

@@ -156,6 +156,10 @@ public final class ImageGalleryController {
         return groupManager;
     }
     
+    public DrawableDB getDatabase(){
+        return db;
+    }
+
     public void setListeningEnabled(boolean enabled) {
         listeningEnabled.set(enabled);
     }
@@ -338,6 +342,7 @@ public final class ImageGalleryController {
         restartWorker();
         historyManager.clear();
         groupManager.setDB(db);
+        db.initializeImageList();
         SummaryTablePane.getDefault().handleCategoryChanged(Collections.emptyList());
     }
 
@@ -352,6 +357,7 @@ public final class ImageGalleryController {
         Platform.runLater(() -> {
             historyManager.clear();
         });
+        Category.clearTagNames();
 
         Toolbar.getDefault().reset();
         groupManager.clear();
@@ -623,9 +629,9 @@ public final class ImageGalleryController {
         @Override
         public void run() {
             try{
-                DrawableFile<?> drawableFile = DrawableFile.create(getFile(), true);
+                DrawableFile<?> drawableFile = DrawableFile.create(getFile(), true, db.isVideoFile(getFile()));
                 db.updateFile(drawableFile);
-            } catch (NullPointerException ex){
+            } catch (NullPointerException | TskCoreException ex){
                 // This is one of the places where we get many errors if the case is closed during processing.
                 // We don't want to print out a ton of exceptions if this is the case.
                 if(Case.isCaseOpen()){
@@ -710,14 +716,14 @@ public final class ImageGalleryController {
                         if (hasMimeType == null) {
                             if (ImageGalleryModule.isSupported(f)) {
                                 //no mime type but supported =>  add as not analyzed
-                                db.insertFile(DrawableFile.create(f, false), tr);
+                                db.insertFile(DrawableFile.create(f, false, db.isVideoFile(f)), tr);
                             } else {
                                 //no mime type, not supported  => remove ( should never get here)
                                 db.removeFile(f.getId(), tr);
                             }
                         } else {
                             if (hasMimeType) {  // supported mimetype => analyzed
-                                db.updateFile(DrawableFile.create(f, true), tr);
+                                db.updateFile(DrawableFile.create(f, true, db.isVideoFile(f)), tr);
                             } else { //unsupported mimtype => analyzed but shouldn't include
                                 db.removeFile(f.getId(), tr);
                             }
@@ -828,7 +834,7 @@ public final class ImageGalleryController {
                         progressHandle.finish();
                         break;
                     }
-                    db.insertFile(DrawableFile.create(f, false), tr);
+                    db.insertFile(DrawableFile.create(f, false, db.isVideoFile(f)), tr);
                     units++;
                     final int prog = units;
                     progressHandle.progress(f.getName(), units);

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/TagUtils.java

@@ -20,7 +20,9 @@ package org.sleuthkit.autopsy.imagegallery;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.logging.Level;
 import javafx.event.ActionEvent;
 import javafx.event.EventHandler;
@@ -92,10 +94,12 @@ public class TagUtils {
     }
 
     public static void fireChange(Collection<Long> ids) {
+        Set<TagUtils.TagListener> listenersCopy = new HashSet<TagUtils.TagListener>(listeners);
         synchronized (listeners) {
-            for (TagListener list : listeners) {
+            listenersCopy.addAll(listeners);
-                list.handleTagsChanged(ids);
+        }
-            }
+        for (TagListener list : listenersCopy) {
+            list.handleTagsChanged(ids);
         }
     }
 

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/AddDrawableTagAction.java

@@ -75,7 +75,11 @@ public class AddDrawableTagAction extends AddTagAction {
     @Override
     public void addTag(TagName tagName, String comment) {
         Set<Long> selectedFiles = new HashSet<>(FileIDSelectionModel.getInstance().getSelected());
+        addTagsToFiles(tagName, comment, selectedFiles);
+    }
         
+    @Override
+    public void addTagsToFiles(TagName tagName, String comment, Set<Long> selectedFiles){    
         new SwingWorker<Void, Void>() {
             
             @Override

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/AddTagAction.java

@@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.imagegallery.actions;
 import org.sleuthkit.autopsy.imagegallery.datamodel.Category;
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
+import java.util.Set;
 import java.util.logging.Level;
 import javafx.event.ActionEvent;
 import javafx.scene.control.Menu;
@@ -66,6 +67,12 @@ abstract class AddTagAction {
      */
     abstract protected void addTag(TagName tagName, String comment);
     
+    /**
+     * Template method to allow derived classes to add the indicated tag and
+     * comment to a list of one or more file IDs.
+     */
+    abstract protected void addTagsToFiles(TagName tagName, String comment, Set<Long> selectedFiles);    
+
     /**
      * Instances of this class implement a context menu user interface for
      * creating or selecting a tag name for a tag and specifying an optional tag

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/CategorizeAction.java

@@ -72,7 +72,11 @@ public class CategorizeAction extends AddTagAction {
     @Override
     public void addTag(TagName tagName, String comment) {
         Set<Long> selectedFiles = new HashSet<>(FileIDSelectionModel.getInstance().getSelected());
+        addTagsToFiles(tagName, comment, selectedFiles);
+    }
       
+    @Override
+    public void addTagsToFiles(TagName tagName, String comment, Set<Long> selectedFiles){    
         //TODO: should this get submitted to controller rather than a swingworker ? -jm
         new SwingWorker<Object, Object>() {
 
@@ -97,9 +101,11 @@ public class CategorizeAction extends AddTagAction {
                             if (ct.getName().getDisplayName().startsWith(Category.CATEGORY_PREFIX)) {
                                 LOGGER.log(Level.INFO, "removing old category from {0}", file.getName());
                                 Case.getCurrentCase().getServices().getTagsManager().deleteContentTag(ct);
+                                controller.getDatabase().decrementCategoryCount(Category.fromDisplayName(ct.getName().getDisplayName()));
                             }
                         }
 
+                        controller.getDatabase().incrementCategoryCount(Category.fromDisplayName(tagName.getDisplayName()));
                         if (tagName != Category.ZERO.getTagName()) { // no tags for cat-0
                             Case.getCurrentCase().getServices().getTagsManager().addContentTag(file, tagName, comment);
                         }

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/Category.java

@@ -66,11 +66,14 @@ public enum Category implements Comparable<Category> {
     private final static Set<CategoryListener> listeners = new HashSet<>();
 
     public static void fireChange(Collection<Long> ids) {
+        Set<CategoryListener> listenersCopy = new HashSet<CategoryListener>(listeners);
         synchronized (listeners) {
-            for (CategoryListener list : listeners) {
+            listenersCopy.addAll(listeners);
-                list.handleCategoryChanged(ids);
-            }
         }
+        for (CategoryListener list : listenersCopy) {
+            list.handleCategoryChanged(ids);
+        }
+      
     }
 
     public static void registerListener(CategoryListener aThis) {
@@ -151,6 +154,18 @@ public enum Category implements Comparable<Category> {
         return menuItem;
     }
     
+    /**
+     * Use when closing a case to make sure everything is re-initialized in the next case.
+     */
+    public static void clearTagNames(){
+        Category.ZERO.tagName = null;
+        Category.ONE.tagName = null;
+        Category.TWO.tagName = null;
+        Category.THREE.tagName = null;
+        Category.FOUR.tagName = null;
+        Category.FIVE.tagName = null;
+    }
+
     public static interface CategoryListener {
 
         public void handleCategoryChanged(Collection<Long> ids);

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java

@@ -36,6 +36,7 @@ import java.util.Set;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
+import javax.annotation.concurrent.GuardedBy;
 import javax.swing.SortOrder;
 import org.apache.commons.lang.StringUtils;
 import org.openide.util.Exceptions;
@@ -43,6 +44,7 @@ import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.imagegallery.FileUpdateEvent;
 import org.sleuthkit.autopsy.imagegallery.ImageGalleryController;
+import org.sleuthkit.autopsy.imagegallery.ImageGalleryModule;
 import org.sleuthkit.autopsy.imagegallery.grouping.GroupKey;
 import org.sleuthkit.autopsy.imagegallery.grouping.GroupManager;
 import org.sleuthkit.autopsy.imagegallery.grouping.GroupSortBy;
@@ -555,8 +557,11 @@ public class DrawableDB {
         if (tr.isClosed()) {
             throw new IllegalArgumentException("can't update database with closed transaction");
         }
+               
         dbWriteLock();
         try {
+            // Update the list of file IDs in memory
+            addImageFileToList(f.getId());
             
             // "INSERT OR IGNORE/ INTO drawable_files (path, name, created_time, modified_time, make, model, analyzed)"
             stmt.setLong(1, f.getId());
@@ -939,7 +944,8 @@ public class DrawableDB {
      */
     private DrawableFile<?> getFileFromID(Long id, boolean analyzed) throws TskCoreException {
         try {
-            return DrawableFile.create(controller.getSleuthKitCase().getAbstractFileById(id), analyzed);
+            AbstractFile f = controller.getSleuthKitCase().getAbstractFileById(id);
+            return DrawableFile.create(f, analyzed, isVideoFile(f));
         } catch (IllegalStateException ex) {
             LOGGER.log(Level.SEVERE, "there is no case open; failed to load file with id: " + id, ex);
             return null;
@@ -956,8 +962,9 @@ public class DrawableDB {
      */
     public DrawableFile<?> getFileFromID(Long id) throws TskCoreException {
         try {
-            return DrawableFile.create(controller.getSleuthKitCase().getAbstractFileById(id),
+            AbstractFile f = controller.getSleuthKitCase().getAbstractFileById(id);
-                    areFilesAnalyzed(Collections.singleton(id)));
+            return DrawableFile.create(f,
+                    areFilesAnalyzed(Collections.singleton(id)), isVideoFile(f));
         } catch (IllegalStateException ex) {
             LOGGER.log(Level.SEVERE, "there is no case open; failed to load file with id: " + id);
             return null;
@@ -1038,7 +1045,8 @@ public class DrawableDB {
             List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(cat.getTagName());
             for (ContentTag ct : contentTags) {
                 if (ct.getContent() instanceof AbstractFile) {
-                    files.add(DrawableFile.create((AbstractFile) ct.getContent(), isFileAnalyzed(ct.getContent().getId())));
+                    files.add(DrawableFile.create((AbstractFile) ct.getContent(), isFileAnalyzed(ct.getContent().getId()),
+                            isVideoFile((AbstractFile) ct.getContent())));
                 }
             }
             return files;
@@ -1085,6 +1093,9 @@ public class DrawableDB {
         dbWriteLock();
 
         try {
+            // Update the list of file IDs in memory
+            removeImageFileFromList(id);
+            
             //"delete from drawable_files where (obj_id = " + id + ")"
             removeFileStmt.setLong(1, id);
             removeFileStmt.executeUpdate();
@@ -1108,6 +1119,138 @@ public class DrawableDB {
         }
     }
     
+    /**
+     * For performance reasons, keep a list of all file IDs currently in the image database.
+     * Otherwise the database is queried many times to retrieve the same data.
+     */
+    @GuardedBy("fileIDlist")
+    private final Set<Long> fileIDlist = new HashSet<>();
+
+    public boolean isImageFile(Long id) {
+        synchronized (fileIDlist) {
+            return fileIDlist.contains(id);
+        }
+    }
+
+    public void addImageFileToList(Long id) {
+        synchronized (fileIDlist) {
+            fileIDlist.add(id);
+        }
+    }
+
+    public void removeImageFileFromList(Long id) {
+        synchronized (fileIDlist) {
+            fileIDlist.remove(id);
+        }
+    }
+    
+    public void initializeImageList(){
+        synchronized (fileIDlist){
+            dbReadLock();
+            try {
+                Statement stmt = con.createStatement();
+                ResultSet analyzedQuery = stmt.executeQuery("select obj_id from drawable_files");
+                while (analyzedQuery.next()) {
+                    addImageFileToList(analyzedQuery.getLong(OBJ_ID));
+                }
+
+            } catch (SQLException ex) {
+                LOGGER.log(Level.WARNING, "problem loading file IDs: ", ex);
+            } finally {
+                dbReadUnlock();
+            }
+        }
+    }
+    
+    /**
+     * For performance reasons, keep current category counts in memory
+     */    
+    @GuardedBy("categoryCounts")
+    private final Map<Category, Integer> categoryCounts = new HashMap<>();    
+    
+    public void incrementCategoryCount(Category cat) throws TskCoreException{
+        synchronized(categoryCounts){
+            int count = getCategoryCount(cat);
+            count++;
+            categoryCounts.put(cat, count);
+        }
+    }
+    
+    public void decrementCategoryCount(Category cat) throws TskCoreException{
+        synchronized(categoryCounts){
+            int count = getCategoryCount(cat);
+            count--;
+            categoryCounts.put(cat, count);            
+        }
+    }
+    
+    public int getCategoryCount(Category cat) throws TskCoreException{
+        synchronized(categoryCounts){
+            if(categoryCounts.containsKey(cat)){
+                return categoryCounts.get(cat);
+            }
+            else{
+                try {
+                    if (cat == Category.ZERO) {
+
+                        // Category Zero (Uncategorized) files will not be tagged as such - 
+                        // this is really just the default setting. So we count the number of files
+                        // tagged with the other categories and subtract from the total.
+                        int allOtherCatCount = 0;
+                        TagName[] tns = {Category.FOUR.getTagName(), Category.THREE.getTagName(), Category.TWO.getTagName(), Category.ONE.getTagName(), Category.FIVE.getTagName()};
+                        for (TagName tn : tns) {
+                            List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(tn);
+                            for (ContentTag ct : contentTags) {
+                                if(ct.getContent() instanceof AbstractFile){
+                                    AbstractFile f = (AbstractFile)ct.getContent();
+                                    if(this.isImageFile(f.getId())){
+                                        allOtherCatCount++;
+                                    }
+                                }
+                            }
+                        } 
+                        categoryCounts.put(cat, this.countAllFiles() - allOtherCatCount);
+                        return categoryCounts.get(cat);
+                    } else {
+
+                        int fileCount = 0;
+                        List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(cat.getTagName());
+                        for (ContentTag ct : contentTags) {
+                            if(ct.getContent() instanceof AbstractFile){
+                                AbstractFile f = (AbstractFile)ct.getContent();
+                                if(this.isImageFile(f.getId())){
+                                    fileCount++;
+                                }
+                            }
+                        }
+                        categoryCounts.put(cat, fileCount);
+                        return fileCount;
+                    }
+                } catch(IllegalStateException ex){
+                    throw new TskCoreException("Case closed while getting files");
+                }
+            }
+        }
+    }
+    
+    /**
+     * For performance reasons, keep the file type in memory
+     */    
+    @GuardedBy("videoFileMap")
+    private final Map<Long, Boolean> videoFileMap = new HashMap<>(); 
+    
+    public boolean isVideoFile(AbstractFile f) throws TskCoreException{
+        synchronized(videoFileMap){
+            if(videoFileMap.containsKey(f.getId())){
+                return videoFileMap.get(f.getId());
+            }
+            
+            boolean isVideo = ImageGalleryModule.isVideoFile(f);
+            videoFileMap.put(f.getId(), isVideo);
+            return isVideo;
+        }
+    }
+
     /**
      * inner class that can reference access database connection
      */

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableFile.java

@@ -68,6 +68,17 @@ public abstract class DrawableFile<T extends AbstractFile> extends AbstractFile
         }
     }
     
+    /**
+     * Skip the database query if we have already determined the file type.
+     */
+    public static DrawableFile<?> create(AbstractFile abstractFileById, boolean analyzed, boolean isVideo) {
+        if (isVideo) {
+            return new VideoFile<>(abstractFileById, analyzed);
+        } else {
+            return new ImageFile<>(abstractFileById, analyzed);
+        }
+    }    
+
     public static DrawableFile<?> create(Long id, boolean analyzed) throws TskCoreException, IllegalStateException {
 
         AbstractFile abstractFileById = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(id);

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/grouping/GroupManager.java

@@ -531,42 +531,7 @@ public class GroupManager implements FileUpdateEvent.FileUpdateListener {
      * @throws TskCoreException 
      */
     public int countFilesWithCategory(Category category) throws TskCoreException {
-        try {
+        return db.getCategoryCount(category);
-            if (category == Category.ZERO) {
-
-                // It is unlikely that Category Zero (Uncategorized) files will be tagged as such - 
-                // this is really just the default setting. So we count the number of files
-                // tagged with the other categories and subtract from the total.
-                int allOtherCatCount = 0;
-                TagName[] tns = {Category.FOUR.getTagName(), Category.THREE.getTagName(), Category.TWO.getTagName(), Category.ONE.getTagName(), Category.FIVE.getTagName()};
-                for (TagName tn : tns) {
-                    List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(tn);
-                    for (ContentTag ct : contentTags) {
-                        if (ct.getContent() instanceof AbstractFile && ImageGalleryModule.isSupportedAndNotKnown((AbstractFile) ct.getContent())) {
-                            allOtherCatCount++;
-                        }
-                    }
-                }
-                return (db.countAllFiles() - allOtherCatCount);
-            } else {
-
-                int fileCount = 0;
-                List<ContentTag> contentTags = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByTagName(category.getTagName());
-                for (ContentTag ct : contentTags) {
-                    if (ct.getContent() instanceof AbstractFile && ImageGalleryModule.isSupportedAndNotKnown((AbstractFile) ct.getContent())) {
-                        fileCount++;
-                    }
-                }
-
-                return fileCount;
-            }
-        } catch (TskCoreException ex) {
-            LOGGER.log(Level.WARNING, "TSK error getting files in Category:" + category.getDisplayName(), ex);
-            throw ex;
-        } catch(IllegalStateException ex){
-            throw new TskCoreException("Case closed while getting files");
-        }
-    
     }
 
     public List<Long> getFileIDsWithTag(TagName tagName) throws TskCoreException {
@@ -738,6 +703,7 @@ public class GroupManager implements FileUpdateEvent.FileUpdateListener {
                     TagUtils.fireChange(fileIDs);
                 }
                 break;
+            
         }
     }
 

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/GroupPane.java

@@ -22,6 +22,8 @@ import com.google.common.collect.ImmutableMap;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Set;
+import java.util.HashSet;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -277,8 +279,8 @@ public class GroupPane extends BorderPane implements GroupView {
         menuItem.setOnAction(new EventHandler<ActionEvent>() {
             @Override
             public void handle(ActionEvent t) {
-                selectAllFiles();
+                Set<Long> fileIdSet = new HashSet<Long>(getGrouping().fileIds());
-                new CategorizeAction().addTag(cat.getTagName(), "");
+                new CategorizeAction().addTagsToFiles(cat.getTagName(), "", fileIdSet);
 
                 grpCatSplitMenu.setText(cat.getDisplayName());
                 grpCatSplitMenu.setOnAction(this);
@@ -292,8 +294,8 @@ public class GroupPane extends BorderPane implements GroupView {
         menuItem.setOnAction(new EventHandler<ActionEvent>() {
             @Override
             public void handle(ActionEvent t) {
-                selectAllFiles();
+                Set<Long> fileIdSet = new HashSet<Long>(getGrouping().fileIds());
-                AddDrawableTagAction.getInstance().addTag(tn, "");
+                AddDrawableTagAction.getInstance().addTagsToFiles(tn, "", fileIdSet);
 
                 grpTagSplitMenu.setText(tn.getDisplayName());
                 grpTagSplitMenu.setOnAction(this);