From 49e7ed70363789fc4cad9eac58738933e21ff81d Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 25 Jul 2018 19:51:57 -0400 Subject: [PATCH 1/7] 1001: Image Gallery for multi-user cases --- Core/nbproject/project.xml | 3 +- .../imagegallery/ImageGalleryController.java | 216 +++++++++++++++--- .../imagegallery/ImageGalleryModule.java | 4 +- .../ImageGalleryTopComponent.java | 27 +++ .../imagegallery/actions/OpenAction.java | 10 +- .../imagegallery/datamodel/DrawableDB.java | 76 ++++++ 6 files changed, 299 insertions(+), 37 deletions(-) diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 57cbc58c25..dba056009d 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -329,6 +329,7 @@ org.sleuthkit.autopsy.guiutils org.sleuthkit.autopsy.healthmonitor org.sleuthkit.autopsy.ingest + org.sleuthkit.autopsy.ingest.events org.sleuthkit.autopsy.keywordsearchservice org.sleuthkit.autopsy.menuactions org.sleuthkit.autopsy.modules.encryptiondetection @@ -499,7 +500,7 @@ ext/xmpcore-5.1.3.jar release/modules/ext/xmpcore-5.1.3.jar - + ext/SparseBitSet-1.1.jar release/modules/ext/SparseBitSet-1.1.jar diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index cca4ec0f93..d54d7014eb 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -23,8 +23,10 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -53,11 +55,13 @@ import javafx.scene.layout.Region; import javafx.scene.layout.StackPane; import javafx.scene.paint.Color; import javax.annotation.Nullable; +import javax.swing.JOptionPane; import javax.swing.SwingUtilities; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; @@ -66,6 +70,7 @@ import org.sleuthkit.autopsy.coreutils.History; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.ThreadConfined; +import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.imagegallery.actions.UndoRedoManager; import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; @@ -77,9 +82,11 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; import org.sleuthkit.autopsy.imagegallery.gui.NoGroupsDialog; import org.sleuthkit.autopsy.imagegallery.gui.Toolbar; import org.sleuthkit.autopsy.ingest.IngestManager; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -213,10 +220,14 @@ public final class ImageGalleryController { listeningEnabled.addListener((observable, oldValue, newValue) -> { try { - //if we just turned on listening and a case is open and that case is not up to date - if (newValue && !oldValue && ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows())) { - //populate the db - queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase)); + // rebuild drawable db automatically only for single-user cases. + // For multiuser cases, we defer DB rebuild till the user actually opens Image Gallery + if (Case.getCurrentCaseThrows().getCaseType() == CaseType.SINGLE_USER_CASE) { + //if we just turned on listening and a case is open and that case is not up to date + if (newValue && !oldValue && ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows())) { + //populate the db + this.rebuildDB(); + } } } catch (NoCurrentCaseException ex) { LOGGER.log(Level.WARNING, "Exception while getting open case.", ex); @@ -386,6 +397,14 @@ public final class ImageGalleryController { } } + /** + * Rebuilds the DrawableDB database. + * + */ + public void rebuildDB() { + queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase)); + } + /** * reset the state of the controller (eg if the case is closed) */ @@ -411,6 +430,57 @@ public final class ImageGalleryController { db = null; } + /** + * Checks if the datasources table in drawable DB is stale. + * + * @return true if datasources table is stale + */ + boolean isDataSourcesTableStale() { + + // no current case open to check + if ((null == db) || (null == sleuthKitCase)) { + return false; + } + + try { + Set knownDataSourceIds= db.getKnownDataSourceIds(); + List dataSources = sleuthKitCase.getDataSources(); + Set caseDataSourceIds = new HashSet<>(); + dataSources.forEach((ds) -> { + caseDataSourceIds.add(ds.getId()); + }); + + return !(knownDataSourceIds.containsAll(caseDataSourceIds) && caseDataSourceIds.containsAll(knownDataSourceIds)); + } + catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Image Gallery failed to check if datasources table is stale.", ex); + return false; + } + + } + + /** + * Update the datasources table in drawable DB. + * + * @return true if data_sources table is stale + */ + void updateDataSourcesTable() { + // no current case open to update + if ((null == db) || (null == sleuthKitCase)) { + return ; + } + + try { + List dataSources = sleuthKitCase.getDataSources(); + dataSources.forEach((ds) -> { + db.insertDataSource(ds.getId()); + }); + } + catch (TskCoreException ex) { + LOGGER.log(Level.WARNING, "Image Gallery failed to update data_sources table.", ex); + } + } + synchronized private void shutDownDBExecutor() { if (dbExecutor != null) { dbExecutor.shutdownNow(); @@ -483,8 +553,8 @@ public final class ImageGalleryController { void onStart() { Platform.setImplicitExit(false); LOGGER.info("setting up ImageGallery listeners"); //NON-NLS - //TODO can we do anything usefull in an InjestJobEventListener? - //IngestManager.getInstance().addIngestJobEventListener((PropertyChangeEvent evt) -> {}); + + IngestManager.getInstance().addIngestJobEventListener( new IngestJobEventListener()); IngestManager.getInstance().addIngestModuleEventListener(new IngestModuleEventListener()); Case.addPropertyChangeListener(new CaseEventListener()); } @@ -683,6 +753,8 @@ public final class ImageGalleryController { final SleuthkitCase tskCase; ProgressHandle progressHandle; + + private boolean taskCompletionStatus; BulkTransferTask(ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) { this.controller = controller; @@ -712,16 +784,22 @@ public final class ImageGalleryController { progressHandle.switchToDeterminate(files.size()); updateProgress(0.0); + + taskCompletionStatus = true; //do in transaction DrawableDB.DrawableTransaction tr = taskDB.beginTransaction(); int workDone = 0; for (final AbstractFile f : files) { - if (isCancelled() || Thread.interrupted()) { + if (isCancelled()) { LOGGER.log(Level.WARNING, "Task cancelled: not all contents may be transfered to drawable database."); //NON-NLS progressHandle.finish(); break; } + + if (Thread.interrupted()) { + LOGGER.log(Level.WARNING, "BulkTransferTask interrupted. Ignoring it to update the contents of drawable database."); //NON-NLS + } processFile(f, tr); @@ -750,10 +828,14 @@ public final class ImageGalleryController { updateMessage(""); updateProgress(-1.0); } - cleanup(true); + cleanup(taskCompletionStatus); } abstract ProgressHandle getInitialProgressHandle(); + + void setTaskCompletionStatus(boolean status) { + taskCompletionStatus = status; + } } /** @@ -774,6 +856,7 @@ public final class ImageGalleryController { @Override protected void cleanup(boolean success) { + controller.updateDataSourcesTable(); controller.setStale(!success); } @@ -783,7 +866,7 @@ public final class ImageGalleryController { } @Override - void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr) { + void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr) throws TskCoreException { final boolean known = f.getKnown() == TskData.FileKnown.KNOWN; if (known) { @@ -794,10 +877,14 @@ public final class ImageGalleryController { if (FileTypeUtils.hasDrawableMIMEType(f)) { //supported mimetype => analyzed taskDB.updateFile(DrawableFile.create(f, true, false), tr); } else { //unsupported mimtype => analyzed but shouldn't include + // if mimetype of the file hasnt been ascertained, ingest might not have completed yet. + if (null == f.getMIMEType()) { + this.setTaskCompletionStatus(false); + } taskDB.removeFile(f.getId(), tr); } } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - throw new RuntimeException(ex); + throw new TskCoreException("Failed to initialize FileTypeDetector.", ex); } } } @@ -890,28 +977,32 @@ public final class ImageGalleryController { AbstractFile file = (AbstractFile) evt.getNewValue(); - if (isListeningEnabled()) { - if (file.isFile()) { - try { - synchronized (ImageGalleryController.this) { - if (ImageGalleryModule.isDrawableAndNotKnown(file)) { - //this file should be included and we don't already know about it from hash sets (NSRL) - queueDBTask(new UpdateFileTask(file, db)); - } else if (FileTypeUtils.getAllSupportedExtensions().contains(file.getNameExtension())) { - //doing this check results in fewer tasks queued up, and faster completion of db update - //this file would have gotten scooped up in initial grab, but actually we don't need it - queueDBTask(new RemoveFileTask(file, db)); + // only process individual files in realtime on the node that is running the ingest + // on a remote node, image files are processed enblock when ingest is complete + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + if (isListeningEnabled()) { + if (file.isFile()) { + try { + synchronized (ImageGalleryController.this) { + if (ImageGalleryModule.isDrawableAndNotKnown(file)) { + //this file should be included and we don't already know about it from hash sets (NSRL) + queueDBTask(new UpdateFileTask(file, db)); + } else if (FileTypeUtils.getAllSupportedExtensions().contains(file.getNameExtension())) { + //doing this check results in fewer tasks queued up, and faster completion of db update + //this file would have gotten scooped up in initial grab, but actually we don't need it + queueDBTask(new RemoveFileTask(file, db)); + } } + } catch (TskCoreException | FileTypeDetector.FileTypeDetectorInitException ex) { + //TODO: What to do here? + LOGGER.log(Level.SEVERE, "Unable to determine if file is drawable and not known. Not making any changes to DB", ex); //NON-NLS + MessageNotifyUtil.Notify.error("Image Gallery Error", + "Unable to determine if file is drawable and not known. Not making any changes to DB. See the logs for details."); } - } catch (TskCoreException | FileTypeDetector.FileTypeDetectorInitException ex) { - //TODO: What to do here? - LOGGER.log(Level.SEVERE, "Unable to determine if file is drawable and not known. Not making any changes to DB", ex); //NON-NLS - MessageNotifyUtil.Notify.error("Image Gallery Error", - "Unable to determine if file is drawable and not known. Not making any changes to DB. See the logs for details."); } + } else { //TODO: keep track of what we missed for later + setStale(true); } - } else { //TODO: keep track of what we missed for later - setStale(true); } break; } @@ -943,12 +1034,14 @@ public final class ImageGalleryController { } break; case DATA_SOURCE_ADDED: - //copy all file data to drawable databse - Content newDataSource = (Content) evt.getNewValue(); - if (isListeningEnabled()) { - queueDBTask(new PrePopulateDataSourceFiles(newDataSource, ImageGalleryController.this, getDatabase(), getSleuthKitCase())); - } else {//TODO: keep track of what we missed for later - setStale(true); + //For a data source added on the local node, prepopulate all file data to drawable database + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + Content newDataSource = (Content) evt.getNewValue(); + if (isListeningEnabled()) { + queueDBTask(new PrePopulateDataSourceFiles(newDataSource, ImageGalleryController.this, getDatabase(), getSleuthKitCase())); + } else {//TODO: keep track of what we missed for later + setStale(true); + } } break; case CONTENT_TAG_ADDED: @@ -966,4 +1059,59 @@ public final class ImageGalleryController { } } } + + + private class IngestJobEventListener implements PropertyChangeListener { + + @NbBundle.Messages({ + "ImageGalleryController.dataSourceAnalyzed.confDlg.msg= A new data source was added and finished ingest.\n" + + "The image / video database may be out of date. " + + "Do you want to update the database with ingest results?\n", + "ImageGalleryController.dataSourceAnalyzed.confDlg.title=Image Gallery" + }) + @Override + public void propertyChange(PropertyChangeEvent evt) { + switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) { + case DATA_SOURCE_ANALYSIS_COMPLETED: + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) { + // A remote node added a new data source and just finished ingest on it. + //drawable db is stale, and if ImageGallery is open, ask user what to do + setStale(true); + + SwingUtilities.invokeLater(() -> { + if (isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { + + int answer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + + switch (answer) { + case JOptionPane.YES_OPTION: + rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + break; //do nothing + } + } + }); + + } else { + // received event from local node + // add the datasource to drawable db + long dsObjId = 0; + DataSourceAnalysisCompletedEvent event = (DataSourceAnalysisCompletedEvent)evt; + if(event.getDataSource() != null) { + dsObjId = event.getDataSource().getId(); + db.insertDataSource(dsObjId); + } else { + LOGGER.log(Level.SEVERE, "DataSourceAnalysisCompletedEvent does not contain a dataSource object"); //NON-NLS + } + } + break; + } + } + + } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index a5f13c1d9a..a6b57e5779 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -86,7 +86,9 @@ public class ImageGalleryModule { public static boolean isDrawableDBStale(Case c) { if (c != null) { String stale = new PerCaseProperties(c).getConfigSetting(ImageGalleryModule.MODULE_NAME, PerCaseProperties.STALE); - return StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : true; + + return ( ImageGalleryController.getDefault().isDataSourcesTableStale() || + (StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : true) ); } else { return false; } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index a6ce70c3ac..0f98b69e89 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -39,6 +39,7 @@ import org.openide.windows.RetainLocation; import org.openide.windows.TopComponent; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.gui.StatusBar; import org.sleuthkit.autopsy.imagegallery.gui.SummaryTablePane; import org.sleuthkit.autopsy.imagegallery.gui.Toolbar; @@ -88,6 +89,32 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl private VBox leftPane; private Scene myScene; + /** + * Returns whether the ImageGallery window is open or not. + * + * @return true, if Image gallery is opened, false otherwise + */ + @ThreadConfined(type = ThreadConfined.ThreadType.JFX) + public static boolean isImageGalleryOpen() { + + final TopComponent tc = WindowManager.getDefault().findTopComponent(PREFERRED_ID); + if (tc != null) { + return tc.isOpened(); + } + return false; + } + + /** + * Returns the top component window. + * + * @return Image gallery top component window + */ + @ThreadConfined(type = ThreadConfined.ThreadType.JFX) + public static TopComponent getTopComponent() { + final TopComponent tc = WindowManager.getDefault().findTopComponent(PREFERRED_ID); + return tc; + } + public static void openTopComponent() { //TODO:eventually move to this model, throwing away everything and rebuilding controller groupmanager etc for each case. // synchronized (OpenTimelineAction.class) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index e10a679792..2b55be7e49 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -172,7 +172,15 @@ public final class OpenAction extends CallableSystemAction { switch (answer) { case JOptionPane.YES_OPTION: - ImageGalleryController.getDefault().setListeningEnabled(true); + + if (currentCase.getCaseType() == Case.CaseType.SINGLE_USER_CASE) { + // toggling listening to ON automatically triggers a rebuild + ImageGalleryController.getDefault().setListeningEnabled(true); + } + else { + ImageGalleryController.getDefault().rebuildDB(); + } + //fall through case JOptionPane.NO_OPTION: ImageGalleryTopComponent.openTopComponent(); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 01185d4288..89376b12a6 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -103,6 +103,8 @@ public final class DrawableDB { private final PreparedStatement insertHashHitStmt; + private final PreparedStatement insertDataSourceStmt; + private final PreparedStatement updateFileStmt; private final PreparedStatement insertFileStmt; @@ -209,6 +211,10 @@ public final class DrawableDB { "INSERT OR IGNORE INTO drawable_files (obj_id , path, name, created_time, modified_time, make, model, analyzed) " //NON-NLS + "VALUES (?,?,?,?,?,?,?,?)"); //NON-NLS + insertDataSourceStmt = prepareStatement( + "INSERT OR IGNORE INTO datasources (ds_obj_id) " //NON-NLS + + "VALUES (?)"); //NON-NLS + removeFileStmt = prepareStatement("DELETE FROM drawable_files WHERE obj_id = ?"); //NON-NLS pathGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE path = ? ", DrawableAttribute.PATH); //NON-NLS @@ -349,6 +355,17 @@ public final class DrawableDB { LOGGER.log(Level.SEVERE, "problem accessing database", ex); //NON-NLS return false; } + + try (Statement stmt = con.createStatement()) { + String sql = "CREATE TABLE if not exists datasources " //NON-NLS + + "( id INTEGER PRIMARY KEY, " //NON-NLS + + " ds_obj_id integer UNIQUE NOT NULL)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "problem creating datasources table", ex); //NON-NLS + return false; + } + try (Statement stmt = con.createStatement()) { String sql = "CREATE TABLE if not exists drawable_files " //NON-NLS + "( obj_id INTEGER PRIMARY KEY, " //NON-NLS @@ -689,6 +706,65 @@ public final class DrawableDB { } } + + /** + * Gets all known data source object ids from data_sources table + * + * @return list of known data source object ids + */ + public Set getKnownDataSourceIds() throws TskCoreException { + Statement statement = null; + ResultSet rs = null; + Set ret = new HashSet<>(); + dbReadLock(); + try { + statement = con.createStatement(); + rs = statement.executeQuery("SELECT ds_obj_id FROM datasources "); //NON-NLS + while (rs.next()) { + ret.add(rs.getLong(1)); + } + } catch (SQLException e) { + throw new TskCoreException("SQLException thrown when calling 'DrawableDB.getKnownDataSourceIds()", e); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "Error closing result set after executing getKnownDataSourceIds", ex); //NON-NLS + } + } + if (statement != null) { + try { + statement.close(); + } catch (SQLException ex) { + LOGGER.log(Level.SEVERE, "Error closing statement after executing getKnownDataSourceIds", ex); //NON-NLS + } + } + dbReadUnlock(); + } + return ret; + } + + + /** + * Insert given data source object id into data_sources table + * + * @param dsObjectId data source object id to insert + */ + public void insertDataSource(long dsObjectId) { + dbWriteLock(); + try { + // "INSERT OR IGNORE/ INTO datasources (ds_obj_id)" + insertDataSourceStmt.setLong(1,dsObjectId); + + insertDataSourceStmt.executeUpdate(); + } catch (SQLException | NullPointerException ex) { + LOGGER.log(Level.SEVERE, "failed to insert/update datasources table", ex); //NON-NLS + } finally { + dbWriteUnlock(); + } + } + public DrawableTransaction beginTransaction() { return new DrawableTransaction(); } From 9a5461fd6c6ebfdca0a6ca05c48bdecd6b8e56d5 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 26 Jul 2018 08:38:48 -0400 Subject: [PATCH 2/7] 1001: Image Gallery for multi-user cases - Address Codacy comment in previous commit. --- .../imagegallery/ImageGalleryController.java | 93 ++++++++++--------- .../ImageGalleryTopComponent.java | 10 +- .../imagegallery/datamodel/DrawableDB.java | 2 - 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index d54d7014eb..f13abdd2d4 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -220,15 +220,15 @@ public final class ImageGalleryController { listeningEnabled.addListener((observable, oldValue, newValue) -> { try { - // rebuild drawable db automatically only for single-user cases. + // if we just turned on listening and a single-user case is open and that case is not up to date, then rebuild it // For multiuser cases, we defer DB rebuild till the user actually opens Image Gallery - if (Case.getCurrentCaseThrows().getCaseType() == CaseType.SINGLE_USER_CASE) { - //if we just turned on listening and a case is open and that case is not up to date - if (newValue && !oldValue && ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows())) { - //populate the db - this.rebuildDB(); - } + if ( newValue && !oldValue && + ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows()) && + (Case.getCurrentCaseThrows().getCaseType() == CaseType.SINGLE_USER_CASE) ) { + //populate the db + this.rebuildDB(); } + } catch (NoCurrentCaseException ex) { LOGGER.log(Level.WARNING, "Exception while getting open case.", ex); } @@ -446,8 +446,8 @@ public final class ImageGalleryController { Set knownDataSourceIds= db.getKnownDataSourceIds(); List dataSources = sleuthKitCase.getDataSources(); Set caseDataSourceIds = new HashSet<>(); - dataSources.forEach((ds) -> { - caseDataSourceIds.add(ds.getId()); + dataSources.forEach((dataSource) -> { + caseDataSourceIds.add(dataSource.getId()); }); return !(knownDataSourceIds.containsAll(caseDataSourceIds) && caseDataSourceIds.containsAll(knownDataSourceIds)); @@ -1060,7 +1060,10 @@ public final class ImageGalleryController { } } - + + /** + * Listener for Ingest Job events. + */ private class IngestJobEventListener implements PropertyChangeListener { @NbBundle.Messages({ @@ -1071,47 +1074,45 @@ public final class ImageGalleryController { }) @Override public void propertyChange(PropertyChangeEvent evt) { - switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) { - case DATA_SOURCE_ANALYSIS_COMPLETED: - if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) { - // A remote node added a new data source and just finished ingest on it. - //drawable db is stale, and if ImageGallery is open, ask user what to do - setStale(true); - - SwingUtilities.invokeLater(() -> { - if (isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { - - int answer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + String eventName = evt.getPropertyName(); + if ( eventName.equals(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED.toString())) { + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) { + // A remote node added a new data source and just finished ingest on it. + //drawable db is stale, and if ImageGallery is open, ask user what to do + setStale(true); - switch (answer) { - case JOptionPane.YES_OPTION: - rebuildDB(); - break; - case JOptionPane.NO_OPTION: - case JOptionPane.CANCEL_OPTION: - break; //do nothing - } + SwingUtilities.invokeLater(() -> { + if (isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { + + int answer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + + switch (answer) { + case JOptionPane.YES_OPTION: + rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + default: + break; //do nothing } - }); - - } else { - // received event from local node - // add the datasource to drawable db - long dsObjId = 0; - DataSourceAnalysisCompletedEvent event = (DataSourceAnalysisCompletedEvent)evt; - if(event.getDataSource() != null) { - dsObjId = event.getDataSource().getId(); - db.insertDataSource(dsObjId); - } else { - LOGGER.log(Level.SEVERE, "DataSourceAnalysisCompletedEvent does not contain a dataSource object"); //NON-NLS } + }); + } else { + // received event from local node + // add the datasource to drawable db + long dsObjId = 0; + DataSourceAnalysisCompletedEvent event = (DataSourceAnalysisCompletedEvent)evt; + if(event.getDataSource() != null) { + dsObjId = event.getDataSource().getId(); + db.insertDataSource(dsObjId); + } else { + LOGGER.log(Level.SEVERE, "DataSourceAnalysisCompletedEvent does not contain a dataSource object"); //NON-NLS } - break; + } } } - } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index 0f98b69e89..fbb3f2b8a4 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -97,9 +97,9 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl @ThreadConfined(type = ThreadConfined.ThreadType.JFX) public static boolean isImageGalleryOpen() { - final TopComponent tc = WindowManager.getDefault().findTopComponent(PREFERRED_ID); - if (tc != null) { - return tc.isOpened(); + final TopComponent topComponent = WindowManager.getDefault().findTopComponent(PREFERRED_ID); + if (topComponent != null) { + return topComponent.isOpened(); } return false; } @@ -111,8 +111,8 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl */ @ThreadConfined(type = ThreadConfined.ThreadType.JFX) public static TopComponent getTopComponent() { - final TopComponent tc = WindowManager.getDefault().findTopComponent(PREFERRED_ID); - return tc; + final TopComponent topComponent = WindowManager.getDefault().findTopComponent(PREFERRED_ID); + return topComponent; } public static void openTopComponent() { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 89376b12a6..29473df5b8 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -47,7 +47,6 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.swing.SortOrder; import org.apache.commons.lang3.StringUtils; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.imagegallery.FileTypeUtils; @@ -63,7 +62,6 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.DBAccessManager; import org.sleuthkit.datamodel.DBAccessQueryCallback; import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; From 36c638514f280a63e80ec0a5792bf6c7fc729f6a Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 26 Jul 2018 09:09:20 -0400 Subject: [PATCH 3/7] 1001: Image gallery for muti-user cases - More Codacy comments addressed. --- .../autopsy/imagegallery/ImageGalleryController.java | 4 ++-- .../autopsy/imagegallery/ImageGalleryTopComponent.java | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index f13abdd2d4..cc3d757ab6 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -472,8 +472,8 @@ public final class ImageGalleryController { try { List dataSources = sleuthKitCase.getDataSources(); - dataSources.forEach((ds) -> { - db.insertDataSource(ds.getId()); + dataSources.forEach((dataSource) -> { + db.insertDataSource(dataSource.getId()); }); } catch (TskCoreException ex) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index fbb3f2b8a4..1add37a2b7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -107,12 +107,11 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl /** * Returns the top component window. * - * @return Image gallery top component window + * @return Image gallery top component window, null if it's not open */ @ThreadConfined(type = ThreadConfined.ThreadType.JFX) public static TopComponent getTopComponent() { - final TopComponent topComponent = WindowManager.getDefault().findTopComponent(PREFERRED_ID); - return topComponent; + return WindowManager.getDefault().findTopComponent(PREFERRED_ID); } public static void openTopComponent() { From 6b11132bedf6caf9c3ef2b20820a1dd624cabd68 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 26 Jul 2018 21:53:03 -0400 Subject: [PATCH 4/7] Modify the extension query in BulkTransferTask to use the extension column instead of substring match on name column. --- .../autopsy/imagegallery/ImageGalleryController.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index cc3d757ab6..42fc603fd4 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -729,12 +729,12 @@ public final class ImageGalleryController { "BulkTask.errPopulating.errMsg=There was an error populating Image Gallery database."}) /* Base abstract class for various methods of copying data into the Image gallery DB */ abstract static private class BulkTransferTask extends BackgroundTask { - + static private final String FILE_EXTENSION_CLAUSE = - "(name LIKE '%." //NON-NLS - + String.join("' OR name LIKE '%.", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS - + "')"; - + "(extension LIKE '" //NON-NLS + + String.join("' OR extension LIKE '", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS + + "') "; + static private final String MIMETYPE_CLAUSE = "(mime_type LIKE '" //NON-NLS + String.join("' OR mime_type LIKE '", FileTypeUtils.getAllSupportedMimeTypes()) //NON-NLS From 834506324458874c459d250c7c0a0e467e8d686d Mon Sep 17 00:00:00 2001 From: Raman Date: Fri, 27 Jul 2018 09:19:18 -0400 Subject: [PATCH 5/7] Mark DB is not stale when a local ingest finishes. --- .../sleuthkit/autopsy/imagegallery/ImageGalleryController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 42fc603fd4..3c45229e78 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -1108,6 +1108,8 @@ public final class ImageGalleryController { if(event.getDataSource() != null) { dsObjId = event.getDataSource().getId(); db.insertDataSource(dsObjId); + // All files for the data source have been analyzed. + setStale(false); } else { LOGGER.log(Level.SEVERE, "DataSourceAnalysisCompletedEvent does not contain a dataSource object"); //NON-NLS } From dd6e3abafedb7e7f2c1afa8ec1f86f634b7c9ae0 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 27 Jul 2018 12:35:49 -0400 Subject: [PATCH 6/7] Update ImageGalleryController.java --- .../sleuthkit/autopsy/imagegallery/ImageGalleryController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 3c45229e78..22ebc8df9e 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -477,7 +477,7 @@ public final class ImageGalleryController { }); } catch (TskCoreException ex) { - LOGGER.log(Level.WARNING, "Image Gallery failed to update data_sources table.", ex); + LOGGER.log(Level.SEVERE, "Image Gallery failed to update data_sources table.", ex); } } From 1d15ac3622e9e9f089265084c277829c3a090f79 Mon Sep 17 00:00:00 2001 From: Raman Date: Mon, 30 Jul 2018 08:04:16 -0400 Subject: [PATCH 7/7] 1001: Image gallery supports multi user cases - addressed review comments on previous commit. --- .../imagegallery/ImageGalleryController.java | 28 ++++++++++--------- .../ImageGalleryTopComponent.java | 2 -- .../imagegallery/actions/OpenAction.java | 5 +++- .../imagegallery/datamodel/DrawableDB.java | 14 ++++++---- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 3c45229e78..5f53ca1bc5 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -438,13 +438,13 @@ public final class ImageGalleryController { boolean isDataSourcesTableStale() { // no current case open to check - if ((null == db) || (null == sleuthKitCase)) { + if ((null == getDatabase()) || (null == getSleuthKitCase())) { return false; } try { - Set knownDataSourceIds= db.getKnownDataSourceIds(); - List dataSources = sleuthKitCase.getDataSources(); + Set knownDataSourceIds= getDatabase().getDataSourceIds(); + List dataSources = getSleuthKitCase().getDataSources(); Set caseDataSourceIds = new HashSet<>(); dataSources.forEach((dataSource) -> { caseDataSourceIds.add(dataSource.getId()); @@ -462,18 +462,17 @@ public final class ImageGalleryController { /** * Update the datasources table in drawable DB. * - * @return true if data_sources table is stale */ - void updateDataSourcesTable() { + private void updateDataSourcesTable() { // no current case open to update - if ((null == db) || (null == sleuthKitCase)) { - return ; + if ((null == getDatabase()) || (null == getSleuthKitCase())) { + return; } try { - List dataSources = sleuthKitCase.getDataSources(); + List dataSources = getSleuthKitCase().getDataSources(); dataSources.forEach((dataSource) -> { - db.insertDataSource(dataSource.getId()); + getDatabase().insertDataSource(dataSource.getId()); }); } catch (TskCoreException ex) { @@ -833,7 +832,7 @@ public final class ImageGalleryController { abstract ProgressHandle getInitialProgressHandle(); - void setTaskCompletionStatus(boolean status) { + protected void setTaskCompletionStatus(boolean status) { taskCompletionStatus = status; } } @@ -874,10 +873,13 @@ public final class ImageGalleryController { } else { try { - if (FileTypeUtils.hasDrawableMIMEType(f)) { //supported mimetype => analyzed + //supported mimetype => analyzed + if ( null != f.getMIMEType() && FileTypeUtils.hasDrawableMIMEType(f)) { taskDB.updateFile(DrawableFile.create(f, true, false), tr); - } else { //unsupported mimtype => analyzed but shouldn't include - // if mimetype of the file hasnt been ascertained, ingest might not have completed yet. + } + else { //unsupported mimtype => analyzed but shouldn't include + + // if mimetype of the file hasn't been ascertained, ingest might not have completed yet. if (null == f.getMIMEType()) { this.setTaskCompletionStatus(false); } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index 1add37a2b7..ce06c52893 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -94,7 +94,6 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl * * @return true, if Image gallery is opened, false otherwise */ - @ThreadConfined(type = ThreadConfined.ThreadType.JFX) public static boolean isImageGalleryOpen() { final TopComponent topComponent = WindowManager.getDefault().findTopComponent(PREFERRED_ID); @@ -109,7 +108,6 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl * * @return Image gallery top component window, null if it's not open */ - @ThreadConfined(type = ThreadConfined.ThreadType.JFX) public static TopComponent getTopComponent() { return WindowManager.getDefault().findTopComponent(PREFERRED_ID); } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 2b55be7e49..2fbc9b3758 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -173,8 +173,11 @@ public final class OpenAction extends CallableSystemAction { switch (answer) { case JOptionPane.YES_OPTION: + // For a single-user case, we favor user experience, and rebuild the database + // as soon as Image Gallery is enabled for the case. + // For a multi-user case, we favor overall performance and user experience, not every user may want to review images, + // so we rebuild the database only when a user launches Image Gallery if (currentCase.getCaseType() == Case.CaseType.SINGLE_USER_CASE) { - // toggling listening to ON automatically triggers a rebuild ImageGalleryController.getDefault().setListeningEnabled(true); } else { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index edb0f8cb29..9adf35312d 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -706,11 +706,11 @@ public final class DrawableDB { /** - * Gets all known data source object ids from data_sources table + * Gets all data source object ids from datasources table * * @return list of known data source object ids */ - public Set getKnownDataSourceIds() throws TskCoreException { + public Set getDataSourceIds() throws TskCoreException { Statement statement = null; ResultSet rs = null; Set ret = new HashSet<>(); @@ -722,20 +722,20 @@ public final class DrawableDB { ret.add(rs.getLong(1)); } } catch (SQLException e) { - throw new TskCoreException("SQLException thrown when calling 'DrawableDB.getKnownDataSourceIds()", e); + throw new TskCoreException("SQLException while getting data source object ids", e); } finally { if (rs != null) { try { rs.close(); } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "Error closing result set after executing getKnownDataSourceIds", ex); //NON-NLS + LOGGER.log(Level.SEVERE, "Error closing resultset", ex); //NON-NLS } } if (statement != null) { try { statement.close(); } catch (SQLException ex) { - LOGGER.log(Level.SEVERE, "Error closing statement after executing getKnownDataSourceIds", ex); //NON-NLS + LOGGER.log(Level.SEVERE, "Error closing statement ", ex); //NON-NLS } } dbReadUnlock(); @@ -745,7 +745,9 @@ public final class DrawableDB { /** - * Insert given data source object id into data_sources table + * Insert given data source object id into datasources table + * + * If the object id exists in the table already, it does nothing. * * @param dsObjectId data source object id to insert */