Merge pull request #3990 from raman-bt/1001-ig-multiuser

1001 ig multiuser

Core/nbproject/project.xml

@@ -329,6 +329,7 @@
                 <package>org.sleuthkit.autopsy.guiutils</package>
                 <package>org.sleuthkit.autopsy.healthmonitor</package>
                 <package>org.sleuthkit.autopsy.ingest</package>
+                <package>org.sleuthkit.autopsy.ingest.events</package>
                 <package>org.sleuthkit.autopsy.keywordsearchservice</package>
                 <package>org.sleuthkit.autopsy.menuactions</package>
                 <package>org.sleuthkit.autopsy.modules.encryptiondetection</package>

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java

@@ -23,8 +23,10 @@ import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.beans.PropertyChangeEvent;
 import java.beans.PropertyChangeListener;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
@@ -53,11 +55,13 @@ import javafx.scene.layout.Region;
 import javafx.scene.layout.StackPane;
 import javafx.scene.paint.Color;
 import javax.annotation.Nullable;
+import javax.swing.JOptionPane;
 import javax.swing.SwingUtilities;
 import org.netbeans.api.progress.ProgressHandle;
 import org.openide.util.Cancellable;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
+import org.sleuthkit.autopsy.casemodule.Case.CaseType;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent;
 import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent;
@@ -66,6 +70,7 @@ import org.sleuthkit.autopsy.coreutils.History;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
 import org.sleuthkit.autopsy.coreutils.ThreadConfined;
+import org.sleuthkit.autopsy.events.AutopsyEvent;
 import org.sleuthkit.autopsy.imagegallery.actions.UndoRedoManager;
 import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager;
 import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
@@ -77,9 +82,11 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState;
 import org.sleuthkit.autopsy.imagegallery.gui.NoGroupsDialog;
 import org.sleuthkit.autopsy.imagegallery.gui.Toolbar;
 import org.sleuthkit.autopsy.ingest.IngestManager;
+import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent;
 import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.Content;
+import org.sleuthkit.datamodel.DataSource;
 import org.sleuthkit.datamodel.SleuthkitCase;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
@@ -213,11 +220,15 @@ public final class ImageGalleryController {
 
         listeningEnabled.addListener((observable, oldValue, newValue) -> {
             try {
-                //if we just turned on listening and a case is open and that case is not up to date
+                // if we just turned on listening and a single-user case is open and that case is not up to date, then rebuild it
-                if (newValue && !oldValue && ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows())) {
+                // For multiuser cases, we defer DB rebuild till the user actually opens Image Gallery
+                if ( newValue && !oldValue && 
+                    ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows()) && 
+                    (Case.getCurrentCaseThrows().getCaseType() == CaseType.SINGLE_USER_CASE) ) {
                     //populate the db
-                    queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase));
+                    this.rebuildDB();
                 }
+
             } catch (NoCurrentCaseException ex) {
                 LOGGER.log(Level.WARNING, "Exception while getting open case.", ex);
             }
@@ -386,6 +397,14 @@ public final class ImageGalleryController {
         }
     }
 
+    /**
+     * Rebuilds the DrawableDB database.
+     * 
+     */
+    public void rebuildDB() {
+          queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase));
+    }
+    
     /**
      * reset the state of the controller (eg if the case is closed)
      */
@@ -411,6 +430,56 @@ public final class ImageGalleryController {
         db = null;
     }
 
+    /**
+     * Checks if the datasources table in drawable DB is stale.
+     * 
+     * @return true if datasources table is stale
+     */
+    boolean isDataSourcesTableStale() {
+        
+        // no current case open to check
+        if ((null == getDatabase()) || (null == getSleuthKitCase())) {
+            return false;
+        }
+        
+        try {
+            Set<Long> knownDataSourceIds= getDatabase().getDataSourceIds();
+            List<DataSource> dataSources = getSleuthKitCase().getDataSources();
+            Set<Long> caseDataSourceIds = new HashSet<>();
+            dataSources.forEach((dataSource) -> {
+                caseDataSourceIds.add(dataSource.getId());
+            });
+            
+            return !(knownDataSourceIds.containsAll(caseDataSourceIds) && caseDataSourceIds.containsAll(knownDataSourceIds));
+        }
+        catch (TskCoreException ex) {
+            LOGGER.log(Level.SEVERE, "Image Gallery failed to check if datasources table is stale.", ex);
+            return false;
+        }
+ 
+    }
+    
+    /**
+     * Update the datasources table in drawable DB.
+     * 
+     */
+    private void updateDataSourcesTable() {
+        // no current case open to update
+        if ((null == getDatabase()) || (null == getSleuthKitCase())) {
+            return;
+        }
+        
+        try {
+            List<DataSource> dataSources = getSleuthKitCase().getDataSources();
+            dataSources.forEach((dataSource) -> {
+                getDatabase().insertDataSource(dataSource.getId());
+            });
+        }
+        catch (TskCoreException ex) {
+            LOGGER.log(Level.SEVERE, "Image Gallery failed to update data_sources table.", ex);
+        }
+    }
+    
     synchronized private void shutDownDBExecutor() {
         if (dbExecutor != null) {
             dbExecutor.shutdownNow();
@@ -483,8 +552,8 @@ public final class ImageGalleryController {
     void onStart() {
         Platform.setImplicitExit(false);
         LOGGER.info("setting up ImageGallery listeners"); //NON-NLS
-        //TODO can we do anything usefull in an InjestJobEventListener?
+       
-        //IngestManager.getInstance().addIngestJobEventListener((PropertyChangeEvent evt) -> {});
+        IngestManager.getInstance().addIngestJobEventListener( new IngestJobEventListener());
         IngestManager.getInstance().addIngestModuleEventListener(new IngestModuleEventListener());
         Case.addPropertyChangeListener(new CaseEventListener());
     }
@@ -661,8 +730,8 @@ public final class ImageGalleryController {
     abstract static private class BulkTransferTask extends BackgroundTask {
      
         static private final String FILE_EXTENSION_CLAUSE =
-                "(name LIKE '%." //NON-NLS
+                "(extension LIKE '" //NON-NLS
-                + String.join("' OR name LIKE '%.", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS
+                + String.join("' OR extension LIKE '", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS
                 + "') ";
         
         static private final String MIMETYPE_CLAUSE =
@@ -684,6 +753,8 @@ public final class ImageGalleryController {
 
         ProgressHandle progressHandle;
         
+        private boolean taskCompletionStatus;
+
         BulkTransferTask(ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
             this.controller = controller;
             this.taskDB = taskDB;
@@ -713,16 +784,22 @@ public final class ImageGalleryController {
 
                 updateProgress(0.0);
                 
+               taskCompletionStatus = true;
+
                 //do in transaction
                 DrawableDB.DrawableTransaction tr = taskDB.beginTransaction();
                 int workDone = 0;
                 for (final AbstractFile f : files) {
-                    if (isCancelled() || Thread.interrupted()) {
+                    if (isCancelled()) {
                         LOGGER.log(Level.WARNING, "Task cancelled: not all contents may be transfered to drawable database."); //NON-NLS
                         progressHandle.finish();
                         break;
                     }
                     
+                    if (Thread.interrupted()) {
+                        LOGGER.log(Level.WARNING, "BulkTransferTask interrupted. Ignoring it to update the contents of drawable database."); //NON-NLS
+                    }
+
                     processFile(f, tr);
 
                     workDone++;
@@ -750,10 +827,14 @@ public final class ImageGalleryController {
                 updateMessage("");
                 updateProgress(-1.0);
             }
-            cleanup(true);
+            cleanup(taskCompletionStatus);
         }
 
         abstract ProgressHandle getInitialProgressHandle();
+        
+        protected void setTaskCompletionStatus(boolean status) {
+            taskCompletionStatus = status;
+        }
     }
 
     /**
@@ -774,6 +855,7 @@ public final class ImageGalleryController {
 
         @Override
         protected void cleanup(boolean success) {
+            controller.updateDataSourcesTable();
             controller.setStale(!success);
         }
 
@@ -783,7 +865,7 @@ public final class ImageGalleryController {
         }
 
         @Override
-        void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr) {
+        void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr) throws TskCoreException {
             final boolean known = f.getKnown() == TskData.FileKnown.KNOWN;
 
             if (known) {
@@ -791,13 +873,20 @@ public final class ImageGalleryController {
             } else {
 
                 try {
-                    if (FileTypeUtils.hasDrawableMIMEType(f)) {  //supported mimetype => analyzed
+                    //supported mimetype => analyzed
+                    if ( null != f.getMIMEType() && FileTypeUtils.hasDrawableMIMEType(f)) {
                         taskDB.updateFile(DrawableFile.create(f, true, false), tr);
-                    } else { //unsupported mimtype => analyzed but shouldn't include
+                    }
+                    else { //unsupported mimtype => analyzed but shouldn't include
+                        
+                        // if mimetype of the file hasn't been ascertained, ingest might not have completed yet.
+                        if (null == f.getMIMEType()) {
+                            this.setTaskCompletionStatus(false);
+                        }
                         taskDB.removeFile(f.getId(), tr);
                     }
                 } catch (FileTypeDetector.FileTypeDetectorInitException ex) {
-                    throw new RuntimeException(ex);
+                    throw new TskCoreException("Failed to initialize FileTypeDetector.", ex);
                 }
             }
         }
@@ -890,6 +979,9 @@ public final class ImageGalleryController {
 
                     AbstractFile file = (AbstractFile) evt.getNewValue();
 
+                    // only process individual files in realtime on the node that is running the ingest
+                    // on a remote node, image files are processed enblock when ingest is complete
+                    if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) {
                         if (isListeningEnabled()) {
                             if (file.isFile()) {
                                 try {
@@ -913,6 +1005,7 @@ public final class ImageGalleryController {
                         } else {   //TODO: keep track of what we missed for later
                             setStale(true);
                         }
+                    }
                     break;
             }
         }
@@ -943,13 +1036,15 @@ public final class ImageGalleryController {
                     }
                     break;
                 case DATA_SOURCE_ADDED:
-                    //copy all file data to drawable databse
+                    //For a data source added on the local node, prepopulate all file data to drawable database
+                    if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) {
                         Content newDataSource = (Content) evt.getNewValue();
                         if (isListeningEnabled()) {
                             queueDBTask(new PrePopulateDataSourceFiles(newDataSource, ImageGalleryController.this, getDatabase(), getSleuthKitCase()));
                         } else {//TODO: keep track of what we missed for later
                             setStale(true);
                         }
+                    }
                     break;
                 case CONTENT_TAG_ADDED:
                     final ContentTagAddedEvent tagAddedEvent = (ContentTagAddedEvent) evt;
@@ -966,4 +1061,62 @@ public final class ImageGalleryController {
             }
         }
     }
+    
+    
+    /**
+     * Listener for Ingest Job events.
+     */
+    private class IngestJobEventListener implements PropertyChangeListener {
+
+        @NbBundle.Messages({
+            "ImageGalleryController.dataSourceAnalyzed.confDlg.msg= A new data source was added and finished ingest.\n" + 
+                "The image / video database may be out of date. " +
+                "Do you want to update the database with ingest results?\n",
+            "ImageGalleryController.dataSourceAnalyzed.confDlg.title=Image Gallery"
+        })
+        @Override
+        public void propertyChange(PropertyChangeEvent evt) {
+            String eventName = evt.getPropertyName();
+            if ( eventName.equals(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED.toString())) {
+                if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) {
+                    // A remote node added a new data source and just finished ingest on it. 
+                    //drawable db is stale, and if ImageGallery is open, ask user what to do
+                    setStale(true);
+
+                    SwingUtilities.invokeLater(() -> {
+                        if (isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) {
+
+                            int answer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), 
+                                    Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(),
+                                    Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), 
+                                    JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE);
+
+                            switch (answer) {
+                                case JOptionPane.YES_OPTION:
+                                     rebuildDB();
+                                     break;
+                                case JOptionPane.NO_OPTION:
+                                case JOptionPane.CANCEL_OPTION:
+                                default:
+                                    break; //do nothing
+                            }
+                        }
+                    });
+                } else { 
+                    // received event from local node
+                    // add the datasource to drawable db
+                    long dsObjId = 0;
+                    DataSourceAnalysisCompletedEvent event = (DataSourceAnalysisCompletedEvent)evt;
+                    if(event.getDataSource() != null) {
+                        dsObjId = event.getDataSource().getId();
+                        db.insertDataSource(dsObjId);
+                        // All files for the data source have been analyzed.
+                        setStale(false);
+                    } else {
+                       LOGGER.log(Level.SEVERE, "DataSourceAnalysisCompletedEvent does not contain a dataSource object"); //NON-NLS
+                    }
+                }
+            }
+        }
+    }
 }

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java

@@ -86,7 +86,9 @@ public class ImageGalleryModule {
     public static boolean isDrawableDBStale(Case c) {
         if (c != null) {
             String stale = new PerCaseProperties(c).getConfigSetting(ImageGalleryModule.MODULE_NAME, PerCaseProperties.STALE);
-            return StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : true;
+            
+            return ( ImageGalleryController.getDefault().isDataSourcesTableStale() || 
+                     (StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : true) );
         } else {
             return false;
         }

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java

@@ -39,6 +39,7 @@ import org.openide.windows.RetainLocation;
 import org.openide.windows.TopComponent;
 import org.openide.windows.WindowManager;
 import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.autopsy.coreutils.ThreadConfined;
 import org.sleuthkit.autopsy.imagegallery.gui.StatusBar;
 import org.sleuthkit.autopsy.imagegallery.gui.SummaryTablePane;
 import org.sleuthkit.autopsy.imagegallery.gui.Toolbar;
@@ -88,6 +89,29 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl
     private VBox leftPane;
     private Scene myScene;
 
+    /**
+     * Returns whether the ImageGallery window is open or not.
+     * 
+     * @return true, if Image gallery is opened, false otherwise
+     */
+    public static boolean isImageGalleryOpen() {
+         
+        final TopComponent topComponent =  WindowManager.getDefault().findTopComponent(PREFERRED_ID);
+        if (topComponent != null) {
+            return topComponent.isOpened();
+        }
+        return false;   
+    }
+    
+    /**
+     * Returns the top component window.
+     * 
+     * @return Image gallery top component window, null if it's not open
+     */
+    public static TopComponent getTopComponent() {
+        return WindowManager.getDefault().findTopComponent(PREFERRED_ID);  
+    }
+    
     public static void openTopComponent() {
         //TODO:eventually move to this model, throwing away everything and rebuilding controller groupmanager etc for each case.
         //        synchronized (OpenTimelineAction.class) {

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java

@@ -172,7 +172,18 @@ public final class OpenAction extends CallableSystemAction {
 
             switch (answer) {
                 case JOptionPane.YES_OPTION:
+                    
+                    // For a single-user case, we favor user experience, and rebuild the database 
+                    // as soon as Image Gallery is enabled for the case.
+                    // For a multi-user case, we favor overall performance and user experience, not every user may want to review images, 
+                    // so we rebuild the database only when a user launches Image Gallery
+                    if (currentCase.getCaseType() == Case.CaseType.SINGLE_USER_CASE) {
                         ImageGalleryController.getDefault().setListeningEnabled(true);
+                    }
+                    else {
+                        ImageGalleryController.getDefault().rebuildDB();
+                    }
+                
                 //fall through
                 case JOptionPane.NO_OPTION:
                     ImageGalleryTopComponent.openTopComponent();

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java

@@ -47,7 +47,6 @@ import javax.annotation.Nonnull;
 import javax.annotation.concurrent.GuardedBy;
 import javax.swing.SortOrder;
 import org.apache.commons.lang3.StringUtils;
-import org.openide.util.Exceptions;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.imagegallery.FileTypeUtils;
@@ -102,6 +101,8 @@ public final class DrawableDB {
 
     private final PreparedStatement insertHashHitStmt;
 
+    private final PreparedStatement insertDataSourceStmt;
+     
     private final PreparedStatement updateFileStmt;
     private final PreparedStatement insertFileStmt;
 
@@ -208,6 +209,10 @@ public final class DrawableDB {
                     "INSERT OR IGNORE INTO drawable_files (obj_id , path, name, created_time, modified_time, make, model, analyzed) " //NON-NLS
                     + "VALUES (?,?,?,?,?,?,?,?)"); //NON-NLS
 
+            insertDataSourceStmt = prepareStatement(
+                    "INSERT OR IGNORE INTO datasources (ds_obj_id) " //NON-NLS
+                    + "VALUES (?)"); //NON-NLS
+             
             removeFileStmt = prepareStatement("DELETE FROM drawable_files WHERE obj_id = ?"); //NON-NLS
 
             pathGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE path  = ? ", DrawableAttribute.PATH); //NON-NLS
@@ -348,6 +353,17 @@ public final class DrawableDB {
             LOGGER.log(Level.SEVERE, "problem accessing database", ex); //NON-NLS
             return false;
         }
+        
+         try (Statement stmt = con.createStatement()) {
+            String sql = "CREATE TABLE if not exists datasources " //NON-NLS
+                    + "( id INTEGER PRIMARY KEY, " //NON-NLS
+                    + " ds_obj_id integer UNIQUE NOT NULL)"; //NON-NLS
+            stmt.execute(sql);
+        } catch (SQLException ex) {
+            LOGGER.log(Level.SEVERE, "problem creating datasources table", ex); //NON-NLS
+            return false;
+        }
+         
         try (Statement stmt = con.createStatement()) {
             String sql = "CREATE TABLE  if not exists drawable_files " //NON-NLS
                     + "( obj_id INTEGER PRIMARY KEY, " //NON-NLS
@@ -688,6 +704,67 @@ public final class DrawableDB {
         }
     }
 
+
+   /**
+     * Gets all data source object ids from datasources table
+     *
+     * @return list of known data source object ids
+     */
+    public Set<Long> getDataSourceIds() throws TskCoreException {
+        Statement statement = null;
+        ResultSet rs = null;
+        Set<Long> ret = new HashSet<>();
+        dbReadLock();
+        try {
+            statement = con.createStatement();
+            rs = statement.executeQuery("SELECT ds_obj_id FROM datasources "); //NON-NLS
+            while (rs.next()) {
+                ret.add(rs.getLong(1));
+            }
+        } catch (SQLException e) {
+            throw new TskCoreException("SQLException while getting data source object ids", e);
+        } finally {
+            if (rs != null) {
+                try {
+                    rs.close();
+                } catch (SQLException ex) {
+                    LOGGER.log(Level.SEVERE, "Error closing resultset", ex); //NON-NLS
+                }
+            }
+            if (statement != null) {
+                try {
+                    statement.close();
+                } catch (SQLException ex) {
+                    LOGGER.log(Level.SEVERE, "Error closing statement ", ex); //NON-NLS
+                }
+            }
+            dbReadUnlock();
+        }
+        return ret;
+    }
+    
+    
+    /**
+     * Insert given data source object id into datasources table
+     * 
+     * If the object id exists in the table already, it does nothing.
+     * 
+     * @param dsObjectId data source object id to insert
+     */
+    public void  insertDataSource(long dsObjectId) {
+        dbWriteLock();
+        try {
+            // "INSERT OR IGNORE/ INTO datasources (ds_obj_id)"
+            insertDataSourceStmt.setLong(1,dsObjectId);
+          
+            insertDataSourceStmt.executeUpdate();
+        } catch (SQLException | NullPointerException ex) {
+            LOGGER.log(Level.SEVERE, "failed to insert/update datasources table", ex); //NON-NLS
+        } finally {
+            dbWriteUnlock();
+        }
+    }
+    
     public DrawableTransaction beginTransaction() {
         return new DrawableTransaction();
     }