Merge branch 'seperate_insert_from_update' into timeline_and_image_analyzer

ImageAnalyzer/src/org/sleuthkit/autopsy/imageanalyzer/AutopsyListener.java

@@ -27,8 +27,10 @@ import org.sleuthkit.autopsy.ingest.IngestManager;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.Content;
 
-/** Singleton aggregator for listeners that hook into case and ingest modules.
+/**
- * This class depends on clients to hook up listeners to Autopsy. */
+ * Singleton aggregator for listeners that hook into case and ingest modules.
+ * This class depends on clients to hook up listeners to Autopsy.
+ */
 public class AutopsyListener {
 
     private static final Logger LOGGER = Logger.getLogger(AutopsyListener.class.getName());
@@ -65,26 +67,22 @@ public class AutopsyListener {
         return instance;
     }
 
-    /** listener for ingest events */
+    /**
+     * listener for ingest events
+     */
     private class IngestJobEventListener implements PropertyChangeListener {
 
         @Override
         synchronized public void propertyChange(PropertyChangeEvent evt) {
             switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) {
-                case COMPLETED:
+                //TODO can we do anything usefull here?
-                    if (controller.isListeningEnabled()) {
-                        if (IngestManager.getInstance().isIngestRunning() == false) {
-                            // @@@ Add some logic to not do this if we've done it in the past second
-                            controller.queueTask(controller.new MarkAllFilesAsAnalyzed());
-                        }
-                    } else {
-                        //TODO can we do anything usefull here?
-                    }
             }
         }
     }
 
-    /** listener for ingest events */
+    /**
+     * listener for ingest events
+     */
     private class IngestModuleEventListener implements PropertyChangeListener {
 
         @Override
@@ -104,15 +102,17 @@ public class AutopsyListener {
                      * by file done event, anyways -jm */
                     break;
                 case FILE_DONE:
-                    /** getOldValue has fileID, getNewValue has {@link Abstractfile}
+                    /**
+                     * getOldValue has fileID, getNewValue has {@link Abstractfile}
                      *
-                     * {@link IngestManager#fireModuleDataEvent(org.sleuthkit.autopsy.ingest.ModuleDataEvent) fireModuleDataEvent} */
+                     * {@link IngestManager#fireModuleDataEvent(org.sleuthkit.autopsy.ingest.ModuleDataEvent) fireModuleDataEvent}
+                     */
                     AbstractFile file = (AbstractFile) evt.getNewValue();
                     if (controller.isListeningEnabled()) {
 
                         if (ImageAnalyzerModule.isSupportedAndNotKnown(file)) {
                             //this file should be included and we don't already know about it from hash sets (NSRL)
-                            controller.queueTask(controller.new UpdateFile(file));
+                            controller.queueTask(controller.new UpdateFileTask(file));
                         } else if (ImageAnalyzerModule.getAllSupportedExtensions().contains(file.getNameExtension())) {
                             //doing this check results in fewer tasks queued up, and faster completion of db update
                             //this file would have gotten scooped up in initial grab, but actually we don't need it
@@ -127,7 +127,9 @@ public class AutopsyListener {
         }
     }
 
-    /** listener for case events */
+    /**
+     * listener for case events
+     */
     private class CaseListener implements PropertyChangeListener {
 
         @Override
@@ -161,5 +163,4 @@ public class AutopsyListener {
         }
     }
 
- 
 }

ImageAnalyzer/src/org/sleuthkit/autopsy/imageanalyzer/ImageAnalyzerController.java

@@ -677,28 +677,14 @@ public final class ImageAnalyzerController implements FileUpdateEvent.FileUpdate
 
     }
 
-    /**
+   
-     * Task to mark all unanalyzed files in the DB as analyzed. Just to make
-     * sure that all are displayed. Added because there were rare cases where
-     * something failed and a file was never marked as analyzed and therefore
-     * never displayed. This task should go into the queue at the end after all
-     * of the update tasks.
-     */
-    class MarkAllFilesAsAnalyzed extends InnerTask {
-
-        @Override
-        public void run() {
-            db.markAllFilesAnalyzed();
-//            checkForGroups();
-        }
-    }
 
     /**
      * task that updates one file in database with results from ingest
      */
-    class UpdateFile extends TaskWithFile {
+    class UpdateFileTask extends TaskWithFile {
 
-        public UpdateFile(AbstractFile f) {
+        public UpdateFileTask(AbstractFile f) {
             super(f);
         }
 
@@ -749,7 +735,7 @@ public final class ImageAnalyzerController implements FileUpdateEvent.FileUpdate
             updateMessage("populating analyzed image/video database");
 
             try {
-                //grap all files with supported mime types
+                //grab all files with supported extension or mime types
                 final List<AbstractFile> files = getSleuthKitCase().findAllFilesWhere(DRAWABLE_QUERY + " or tsk_files.obj_id in (select tsk_files.obj_id from tsk_files , blackboard_artifacts,  blackboard_attributes"
                         + " where  blackboard_artifacts.obj_id = tsk_files.obj_id"
                         + " and blackboard_attributes.artifact_id = blackboard_artifacts.artifact_id"
@@ -777,15 +763,15 @@ public final class ImageAnalyzerController implements FileUpdateEvent.FileUpdate
                     } else {
                         if (hasMimeType == null) {
                             if (ImageAnalyzerModule.isSupported(f)) {
-                                //no mime type but supported => not add as not analyzed
+                                //no mime type but supported =>  add as not analyzed
-                                db.updatefile(DrawableFile.create(f, false), tr);
+                                db.insertFile(DrawableFile.create(f, false), tr);
                             } else {
-                                //no mime type, not supported  => remove ( how dd we get here)
+                                //no mime type, not supported  => remove ( should never get here)
                                 db.removeFile(f.getId(), tr);
                             }
                         } else {
                             if (hasMimeType) {  // supported mimetype => analyzed
-                                db.updatefile(DrawableFile.create(f, true), tr);
+                                db.updateFile(DrawableFile.create(f, true), tr);
                             } else { //unsupported mimtype => analyzed but shouldn't include
                                 db.removeFile(f.getId(), tr);
                             }
@@ -873,7 +859,7 @@ public final class ImageAnalyzerController implements FileUpdateEvent.FileUpdate
                         progressHandle.finish();
                         break;
                     }
-                    db.updatefile(DrawableFile.create(f, false), tr);
+                    db.insertFile(DrawableFile.create(f, false), tr);
                     units++;
                     final int prog = units;
                     progressHandle.progress(f.getName(), units);

ImageAnalyzer/src/org/sleuthkit/autopsy/imageanalyzer/datamodel/DrawableDB.java

@@ -91,7 +91,8 @@ public class DrawableDB {
 
     private final PreparedStatement insertHashHitStmt;
 
-    private final PreparedStatement insertFileStmt;
+    private final PreparedStatement updateFileStmt;
+    private PreparedStatement insertFileStmt;
 
     private final PreparedStatement pathGroupStmt;
 
@@ -191,9 +192,13 @@ public class DrawableDB {
         this.dbPath = dbPath;
 
         if (initializeDB()) {
-            insertFileStmt = prepareStatement(
+            updateFileStmt = prepareStatement(
                     "INSERT OR REPLACE INTO drawable_files (obj_id , path, name, created_time, modified_time, make, model, analyzed) "
                     + "VALUES (?,?,?,?,?,?,?,?)");
+            insertFileStmt = prepareStatement(
+                    "INSERT OR IGNORE INTO drawable_files (obj_id , path, name, created_time, modified_time, make, model, analyzed) "
+                    + "VALUES (?,?,?,?,?,?,?,?)");
+
             removeFileStmt = prepareStatement("delete from drawable_files where obj_id = ?");
 
             pathGroupStmt = prepareStatement("select obj_id , analyzed from drawable_files where  path  = ? ", DrawableAttribute.PATH);
@@ -533,20 +538,25 @@ public class DrawableDB {
 
     public void updateFile(DrawableFile<?> f) {
         DrawableTransaction trans = beginTransaction();
-        updatefile(f, trans);
+        updateFile(f, trans);
         commitTransaction(trans, true);
     }
 
-    /**
+    public void insertFile(DrawableFile<?> f) {
-     * use transactions to update files
+        DrawableTransaction trans = beginTransaction();
-     *
+        insertFile(f, trans);
-     * @param f
+        commitTransaction(trans, true);
-     * @param tr
+    }
-     *
+
-     *
+    public void insertFile(DrawableFile<?> f, DrawableTransaction tr) {
-     *
+        insertOrUpdateFile(f, tr, insertFileStmt);
-     */
+    }
-    public void updatefile(DrawableFile<?> f, DrawableTransaction tr) {
+
+    public void updateFile(DrawableFile<?> f, DrawableTransaction tr) {
+        insertOrUpdateFile(f, tr, updateFileStmt);
+    }
+
+    private void insertOrUpdateFile(DrawableFile<?> f, DrawableTransaction tr, PreparedStatement stmt) {
 
         //TODO:      implement batch version -jm
         if (tr.isClosed()) {
@@ -555,16 +565,16 @@ public class DrawableDB {
         dbWriteLock();
         try {
 
-            // "INSERT OR REPLACE INTO drawable_files (path, name, created_time, modified_time, make, model, analyzed)"
+            // "INSERT OR IGNORE/ INTO drawable_files (path, name, created_time, modified_time, make, model, analyzed)"
-            insertFileStmt.setLong(1, f.getId());
+            stmt.setLong(1, f.getId());
-            insertFileStmt.setString(2, f.getDrawablePath());
+            stmt.setString(2, f.getDrawablePath());
-            insertFileStmt.setString(3, f.getName());
+            stmt.setString(3, f.getName());
-            insertFileStmt.setLong(4, f.getCrtime());
+            stmt.setLong(4, f.getCrtime());
-            insertFileStmt.setLong(5, f.getMtime());
+            stmt.setLong(5, f.getMtime());
-            insertFileStmt.setString(6, f.getMake());
+            stmt.setString(6, f.getMake());
-            insertFileStmt.setString(7, f.getModel());
+            stmt.setString(7, f.getModel());
-            insertFileStmt.setInt(8, f.isAnalyzed() ? 1 : 0);
+            stmt.setBoolean(8, f.isAnalyzed());
-            insertFileStmt.executeUpdate();
+            stmt.executeUpdate();
 
             final Collection<String> hashSetNames = DrawableAttribute.HASHSET.getValue(f);
 
@@ -602,7 +612,7 @@ public class DrawableDB {
             tr.addUpdatedFile(f.getId());
 
         } catch (SQLException ex) {
-            LOGGER.log(Level.SEVERE, "failed to update file" + f.getName(), ex);
+            LOGGER.log(Level.SEVERE, "failed to insert/update file" + f.getName(), ex);
         } finally {
             dbWriteUnlock();
         }
@@ -915,7 +925,7 @@ public class DrawableDB {
             }
 
             statement.setObject(1, key.getValue());
-   
+
             try (ResultSet valsResults = statement.executeQuery()) {
                 while (valsResults.next()) {
                     files.add(getFileFromID(valsResults.getLong(OBJ_ID), valsResults.getBoolean(ANALYZED)));
@@ -1003,41 +1013,7 @@ public class DrawableDB {
         return valsResults == 1;
     }
 
-    /**
+   
-     * Mark all un analyzed files as analyzed.
-     *
-     * TODO: This is a hack we only do because their is a bug that even after
-     * ingest is done, their are sometimes still files that are not marked as
-     * analyzed. Ultimately we should track down the underlying bug. -jm
-     *
-     * @return the ids of files that we marked as analyzed
-     */
-    public ArrayList<Long> markAllFilesAnalyzed() {
-        DrawableTransaction trans = beginTransaction();
-        ArrayList<Long> ids = new ArrayList<>();
-        dbWriteLock();
-        try (Statement statement = con.createStatement();
-                ResultSet executeQuery = statement.executeQuery("select obj_id from drawable_files where analyzed = 0")) {
-
-            while (executeQuery.next()) {
-                ids.add(executeQuery.getLong("obj_id"));
-            }
-
-            if (ids.isEmpty() == false) {
-                Logger.getAnonymousLogger().log(Level.INFO, "marking as analyzed " + ids);
-                statement.executeUpdate("update drawable_files set analyzed = 1 where obj_id in (" + StringUtils.join(ids, ",") + ")");
-                trans.updatedFiles.addAll(ids);
-            }
-        } catch (SQLException ex) {
-            LOGGER.log(Level.WARNING, "failed to mark files as analyzed", ex);
-        } finally {
-            dbWriteUnlock();
-        }
-
-        trans.commit(Boolean.TRUE);
-
-        return ids;
-    }
 
     public class MultipleTransactionException extends IllegalStateException {
 

ImageAnalyzer/src/org/sleuthkit/autopsy/imageanalyzer/datamodel/DrawableFile.java

@@ -60,21 +60,21 @@ import org.sleuthkit.datamodel.TskCoreException;
  */
 public abstract class DrawableFile<T extends AbstractFile> extends AbstractFile {
 
-    public static DrawableFile<?> create(AbstractFile abstractFileById, boolean b) {
+    public static DrawableFile<?> create(AbstractFile abstractFileById, boolean analyzed) {
         if (ImageAnalyzerModule.isVideoFile(abstractFileById)) {
-            return new VideoFile<>(abstractFileById, b);
+            return new VideoFile<>(abstractFileById, analyzed);
         } else {
-            return new ImageFile<>(abstractFileById, b);
+            return new ImageFile<>(abstractFileById, analyzed);
         }
     }
 
-    public static DrawableFile<?> create(Long id, boolean b) throws TskCoreException, IllegalStateException {
+    public static DrawableFile<?> create(Long id, boolean analyzed) throws TskCoreException, IllegalStateException {
 
         AbstractFile abstractFileById = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(id);
         if (ImageAnalyzerModule.isVideoFile(abstractFileById)) {
-            return new VideoFile<>(abstractFileById, b);
+            return new VideoFile<>(abstractFileById, analyzed);
         } else {
-            return new ImageFile<>(abstractFileById, b);
+            return new ImageFile<>(abstractFileById, analyzed);
         }
     }
 
@@ -285,8 +285,6 @@ public abstract class DrawableFile<T extends AbstractFile> extends AbstractFile
         }
     }
 
-  
-
     public abstract Image getIcon();
 
     public void setAnalyzed(Boolean analyzed) {

ImageAnalyzer/src/org/sleuthkit/autopsy/imageanalyzer/gui/GroupPane.java

@@ -90,7 +90,6 @@ import org.controlsfx.control.GridCell;
 import org.controlsfx.control.GridView;
 import org.controlsfx.control.SegmentedButton;
 import org.controlsfx.control.action.ActionUtils;
-import org.openide.util.Exceptions;
 import org.openide.util.Lookup;
 import org.openide.util.actions.Presenter;
 import org.openide.windows.TopComponent;
@@ -406,8 +405,7 @@ public class GroupPane extends BorderPane implements GroupView {
             grpTagSplitMenu.setText(TagUtils.getFollowUpTagName().getDisplayName());
             grpTagSplitMenu.setOnAction(createGrpTagMenuItem(TagUtils.getFollowUpTagName()).getOnAction());
         } catch (TskCoreException tskCoreException) {
-            LOGGER.log(Level.WARNING, "failed to load FollowUpTagName", tskCoreException.getLocalizedMessage());
+            LOGGER.log(Level.WARNING, "failed to load FollowUpTagName", tskCoreException);
-            Exceptions.printStackTrace(tskCoreException);
         }
         grpTagSplitMenu.setGraphic(new ImageView(DrawableAttribute.TAGS.getIcon()));
         grpTagSplitMenu.getItems().setAll(grpTagMenues);
@@ -629,8 +627,11 @@ public class GroupPane extends BorderPane implements GroupView {
      */
     void setViewState(GroupViewState viewState) {
         if (viewState == null) {
-            setCenter(null);
+            Platform.runLater(() -> {
-            groupLabel.setText(null);
+                setCenter(null);
+                groupLabel.setText(null);
+            });
+            
         } else {
             if (this.grouping.get() != viewState.getGroup()) {
                 this.grouping.set(viewState.getGroup());