From 4459288126f9e4bb68b2901d0155f0f8867b555a Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 20 Sep 2018 23:26:32 -0400 Subject: [PATCH 1/8] initial caching commit --- .../imagegallery/ImageGalleryController.java | 5 + .../imagegallery/datamodel/DrawableDB.java | 116 +++++++++++++++--- 2 files changed, 102 insertions(+), 19 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index c983e6a721..4055d143c8 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -50,10 +50,12 @@ import javax.annotation.Nonnull; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.History; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; @@ -70,6 +72,7 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; @@ -728,10 +731,12 @@ public final class ImageGalleryController { CopyAnalyzedFiles(long dataSourceObjId, ImageGalleryController controller) { super(dataSourceObjId, controller); + taskDB.buildFileMetaDataCache(); } @Override protected void cleanup(boolean success) { + taskDB.freeFileMetaDataCache(); // at the end of the task, set the stale status based on the // cumulative status of all data sources controller.setStale(controller.isDataSourcesTableStale()); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index f779978dfb..50a7e082c9 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -51,6 +51,7 @@ import javax.annotation.concurrent.GuardedBy; import javax.swing.SortOrder; import static org.apache.commons.lang3.ObjectUtils.notEqual; import org.apache.commons.lang3.StringUtils; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.DhsImageCategory; @@ -766,6 +767,69 @@ public final class DrawableDB { public void updateFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { insertOrUpdateFile(f, tr, updateFileStmt, caseDbTransaction); } + + Set hasTagCache = new HashSet<>(); + Set hasHashCache = new HashSet<>(); + Set hasExifCache = new HashSet<>(); + boolean areCachesLoaded = false; + public void buildFileMetaDataCache() { + try { + // tag tags + try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM content_tags")) { + ResultSet rs = dbQuery.getResultSet(); + while (rs.next()) { + long id = rs.getLong("obj_id"); + hasTagCache.add(id); + } + + } catch (SQLException ex) { + Exceptions.printStackTrace(ex); + } + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); + } + + try { + // hash sets + try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM blackboard_artifacts WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID())) { + ResultSet rs = dbQuery.getResultSet(); + while (rs.next()) { + long id = rs.getLong("obj_id"); + hasHashCache.add(id); + } + + } catch (SQLException ex) { + Exceptions.printStackTrace(ex); + } + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); + } + + try { + // EXIF + try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM blackboard_artifacts WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID())) { + ResultSet rs = dbQuery.getResultSet(); + while (rs.next()) { + long id = rs.getLong("obj_id"); + hasExifCache.add(id); + } + + } catch (SQLException ex) { + Exceptions.printStackTrace(ex); + } + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); + } + + areCachesLoaded = true; + } + + public void freeFileMetaDataCache() { + areCachesLoaded = false; + hasTagCache.clear(); + hasHashCache.clear(); + hasExifCache.clear(); + } /** * Update (or insert) a file in(to) the drawable db. Weather this is an @@ -790,6 +854,7 @@ public final class DrawableDB { try { // "INSERT OR IGNORE/ INTO drawable_files (obj_id, data_source_obj_id, path, name, created_time, modified_time, make, model, analyzed)" stmt.setLong(1, f.getId()); + // @@@ Should be able to get ID directly from abstract file... stmt.setLong(2, f.getAbstractFile().getDataSource().getId()); stmt.setString(3, f.getDrawablePath()); stmt.setString(4, f.getName()); @@ -802,39 +867,52 @@ public final class DrawableDB { // Update the list of file IDs in memory addImageFileToList(f.getId()); - try { - for (String name : f.getHashSetNames()) { + if ((!areCachesLoaded) || (hasHashCache.contains(f.getId()))) { + try { + for (String name : f.getHashSetNames()) { - // "insert or ignore into hash_sets (hash_set_name) values (?)" - insertHashSetStmt.setString(1, name); - insertHashSetStmt.executeUpdate(); + // "insert or ignore into hash_sets (hash_set_name) values (?)" + insertHashSetStmt.setString(1, name); + insertHashSetStmt.executeUpdate(); - //TODO: use nested select to get hash_set_id rather than seperate statement/query - //"select hash_set_id from hash_sets where hash_set_name = ?" - selectHashSetStmt.setString(1, name); - try (ResultSet rs = selectHashSetStmt.executeQuery()) { - while (rs.next()) { - int hashsetID = rs.getInt("hash_set_id"); //NON-NLS - //"insert or ignore into hash_set_hits (hash_set_id, obj_id) values (?,?)"; - insertHashHitStmt.setInt(1, hashsetID); - insertHashHitStmt.setLong(2, f.getId()); - insertHashHitStmt.executeUpdate(); - break; + //TODO: use nested select to get hash_set_id rather than seperate statement/query + //"select hash_set_id from hash_sets where hash_set_name = ?" + selectHashSetStmt.setString(1, name); + try (ResultSet rs = selectHashSetStmt.executeQuery()) { + while (rs.next()) { + int hashsetID = rs.getInt("hash_set_id"); //NON-NLS + //"insert or ignore into hash_set_hits (hash_set_id, obj_id) values (?,?)"; + insertHashHitStmt.setInt(1, hashsetID); + insertHashHitStmt.setLong(2, f.getId()); + insertHashHitStmt.executeUpdate(); + break; + } } } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "failed to insert/update hash hits for file" + f.getContentPathSafe(), ex); //NON-NLS } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "failed to insert/update hash hits for file" + f.getContentPathSafe(), ex); //NON-NLS } //and update all groups this file is in for (DrawableAttribute attr : DrawableAttribute.getGroupableAttrs()) { + if (attr == DrawableAttribute.TAGS) { + if ((!areCachesLoaded) || (hasTagCache.contains(f.getId()) == false)) { + continue; + } + } + else if (attr == DrawableAttribute.MAKE || attr == DrawableAttribute.MODEL) { + if ((!areCachesLoaded) || (hasExifCache.contains(f.getId()) == false)) { + continue; + } + } Collection> vals = attr.getValue(f); for (Comparable val : vals) { if (null != val) { if (attr == DrawableAttribute.PATH) { insertGroup(f.getAbstractFile().getDataSource().getId(), val.toString(), attr, caseDbTransaction); - } else { + } + else { insertGroup(val.toString(), attr, caseDbTransaction); } } From a5a747fbe7bd91343e377c615cc98abc0b632be8 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 20 Sep 2018 23:46:46 -0400 Subject: [PATCH 2/8] Added comment --- .../org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index f433990296..83f4772081 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -773,6 +773,7 @@ public final class DrawableDB { insertOrUpdateFile(f, tr, updateFileStmt, caseDbTransaction); } + // WORK IN PROGRESS Set hasTagCache = new HashSet<>(); Set hasHashCache = new HashSet<>(); Set hasExifCache = new HashSet<>(); From 4f3f58a70526880ce6484cd924fa4452f32c36af Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 21 Sep 2018 16:36:28 -0400 Subject: [PATCH 3/8] use caches for making drawablefile. Make more specific messages when there are multiple data sources --- .../imagegallery/ImageGalleryController.java | 4 ++-- .../ImageGalleryTopComponent.java | 2 +- .../imagegallery/datamodel/DrawableDB.java | 22 +++++++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 4055d143c8..17fcd446e7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -638,7 +638,7 @@ public final class ImageGalleryController { public void run() { progressHandle = getInitialProgressHandle(); progressHandle.start(); - updateMessage(Bundle.CopyAnalyzedFiles_populatingDb_status()); + updateMessage(Bundle.CopyAnalyzedFiles_populatingDb_status() + " (Data Source " + dataSourceObjId + ")" ); DrawableDB.DrawableTransaction drawableDbTransaction = null; CaseDbTransaction caseDbTransaction = null; @@ -675,7 +675,7 @@ public final class ImageGalleryController { progressHandle.finish(); progressHandle = ProgressHandle.createHandle(Bundle.BulkTask_committingDb_status()); - updateMessage(Bundle.BulkTask_committingDb_status()); + updateMessage(Bundle.BulkTask_committingDb_status() + " (Data Source " + dataSourceObjId + ")" ); updateProgress(1.0); progressHandle.start(); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index a1238185ea..542b90b957 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -183,7 +183,7 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl Map dataSourceNames = new HashMap<>(); dataSourceNames.put("All", null); - dataSources.forEach(dataSource -> dataSourceNames.put(dataSource.getName(), dataSource)); + dataSources.forEach(dataSource -> dataSourceNames.put(dataSource.getName() + " (ID: " + dataSource.getId() + ")", dataSource)); Platform.runLater(() -> { ChoiceDialog datasourceDialog = new ChoiceDialog<>(null, dataSourceNames.keySet()); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 83f4772081..56327fee07 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -778,7 +778,11 @@ public final class DrawableDB { Set hasHashCache = new HashSet<>(); Set hasExifCache = new HashSet<>(); boolean areCachesLoaded = false; - public void buildFileMetaDataCache() { + int cacheUserCount = 0; + synchronized public void buildFileMetaDataCache() { + cacheUserCount++; + if (areCachesLoaded == true) + return; try { // tag tags try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM content_tags")) { @@ -830,7 +834,11 @@ public final class DrawableDB { areCachesLoaded = true; } - public void freeFileMetaDataCache() { + synchronized public void freeFileMetaDataCache() { + // dont' free these if there is another task still using them + if (--cacheUserCount > 0) + return; + areCachesLoaded = false; hasTagCache.clear(); hasHashCache.clear(); @@ -866,8 +874,14 @@ public final class DrawableDB { stmt.setString(4, f.getName()); stmt.setLong(5, f.getCrtime()); stmt.setLong(6, f.getMtime()); - stmt.setString(7, f.getMake()); - stmt.setString(8, f.getModel()); + if ((areCachesLoaded) && (hasExifCache.contains(f.getId()) == false)) { + stmt.setString(7, ""); + stmt.setString(8, ""); + } + else { + stmt.setString(7, f.getMake()); + stmt.setString(8, f.getModel()); + } stmt.setBoolean(9, f.isAnalyzed()); stmt.executeUpdate(); // Update the list of file IDs in memory From 7ed9bfd91ddfb98a0b1750ba5f6541145b1f9371 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 26 Sep 2018 11:12:29 -0400 Subject: [PATCH 4/8] Change order of opening TC, ensure group manager has only relevant groups. Updated Comments --- .../imagegallery/ImageGalleryController.java | 23 ++-- .../ImageGalleryTopComponent.java | 38 +++++- .../imagegallery/datamodel/DrawableDB.java | 53 +++++--- .../datamodel/HashSetManager.java | 2 +- .../datamodel/grouping/GroupManager.java | 121 +++++++++++------- 5 files changed, 159 insertions(+), 78 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 17fcd446e7..802db10a35 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -653,6 +653,7 @@ public final class ImageGalleryController { taskCompletionStatus = true; int workDone = 0; + // Cycle through all of the files returned and call processFile on each //do in transaction drawableDbTransaction = taskDB.beginTransaction(); caseDbTransaction = tskCase.beginTransaction(); @@ -680,6 +681,7 @@ public final class ImageGalleryController { progressHandle.start(); caseDbTransaction.commit(); + // pass true so that groupmanager is notified of the changes taskDB.commitTransaction(drawableDbTransaction, true); } catch (TskCoreException ex) { @@ -749,20 +751,19 @@ public final class ImageGalleryController { if (known) { taskDB.removeFile(f.getId(), tr); //remove known files } else { - try { + // if mimetype of the file hasn't been ascertained, ingest might not have completed yet. + if (null == f.getMIMEType()) { + // set to false to force the DB to be marked as stale + this.setTaskCompletionStatus(false); + } //supported mimetype => analyzed - if (null != f.getMIMEType() && FileTypeUtils.hasDrawableMIMEType(f)) { + else if (FileTypeUtils.hasDrawableMIMEType(f)) { taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction); - } else { - // if mimetype of the file hasn't been ascertained, ingest might not have completed yet. - if (null == f.getMIMEType()) { - // set to false to force the DB to be marked as stale - this.setTaskCompletionStatus(false); - } else { - //unsupported mimtype => analyzed but shouldn't include - taskDB.removeFile(f.getId(), tr); - } + } + //unsupported mimtype => analyzed but shouldn't include + else { + taskDB.removeFile(f.getId(), tr); } } catch (FileTypeDetector.FileTypeDetectorInitException ex) { throw new TskCoreException("Failed to initialize FileTypeDetector.", ex); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index 542b90b957..703397a116 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -118,6 +118,8 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl private Node infoOverlay; private final Region infoOverLayBackground = new TranslucentRegion(); + + /** * Returns whether the ImageGallery window is open or not. @@ -142,6 +144,11 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl return WindowManager.getDefault().findTopComponent(PREFERRED_ID); } + /** + * NOTE: This usually gets called on the EDT + * + * @throws NoCurrentCaseException + */ @Messages({ "ImageGalleryTopComponent.openTopCommponent.chooseDataSourceDialog.headerText=Choose a data source to view.", "ImageGalleryTopComponent.openTopCommponent.chooseDataSourceDialog.contentText=Data source:", @@ -149,24 +156,35 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl "ImageGalleryTopComponent.openTopCommponent.chooseDataSourceDialog.titleText=Image Gallery",}) public static void openTopComponent() throws NoCurrentCaseException { + // This creates the top component and adds the UI widgets if it has not yet been opened final TopComponent topComponent = WindowManager.getDefault().findTopComponent(PREFERRED_ID); if (topComponent == null) { return; } - topComponentInitialized = true; + if (topComponent.isOpened()) { showTopComponent(topComponent); return; } + + // Wait until the FX UI has been created. This way, we can always + // show the gray progress screen + // TODO: do this in a more elegant way. + while (topComponentInitialized == false) {} - List dataSources = Collections.emptyList(); ImageGalleryController controller = ImageGalleryModule.getController(); ((ImageGalleryTopComponent) topComponent).setController(controller); + + // Display the UI so taht they can see the progress screen + showTopComponent(topComponent); + + List dataSources = Collections.emptyList(); try { dataSources = controller.getSleuthKitCase().getDataSources(); } catch (TskCoreException tskCoreException) { logger.log(Level.SEVERE, "Unable to get data sourcecs.", tskCoreException); } + GroupManager groupManager = controller.getGroupManager(); synchronized (groupManager) { if (dataSources.size() <= 1 @@ -175,8 +193,6 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl * set to something other than path , don't both to ask for * datasource */ groupManager.regroup(null, groupManager.getGroupBy(), groupManager.getSortBy(), groupManager.getSortOrder(), true); - - showTopComponent(topComponent); return; } } @@ -198,7 +214,6 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl synchronized (groupManager) { groupManager.regroup(dataSource, groupManager.getGroupBy(), groupManager.getSortBy(), groupManager.getSortOrder(), true); } - SwingUtilities.invokeLater(() -> showTopComponent(topComponent)); }); } @@ -266,6 +281,9 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl controller.regroupDisabledProperty().addListener((Observable observable) -> checkForGroups()); controller.getGroupManager().getAnalyzedGroups().addListener((Observable observable) -> Platform.runLater(() -> checkForGroups())); + topComponentInitialized = true; + + // This will cause the UI to show the progress dialog Platform.runLater(() -> checkForGroups()); } }); @@ -329,6 +347,8 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl * Check if there are any fully analyzed groups available from the * GroupManager and remove blocking progress spinners if there are. If there * aren't, add a blocking progress spinner with appropriate message. + * + * This gets called when any group becomes analyzed and when started. */ @ThreadConfined(type = ThreadConfined.ThreadType.JFX) @NbBundle.Messages({ @@ -345,11 +365,14 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl private void checkForGroups() { GroupManager groupManager = controller.getGroupManager(); + // if there are groups to display, then display them + // @@@ Need to check timing on this and make sure we have only groups for the selected DS. Seems like rebuild can cause groups to be created for a DS that is not later selected... if (isNotEmpty(groupManager.getAnalyzedGroups())) { clearNotification(); return; } + // display a message based on if ingest is running and/or listening if (IngestManager.getInstance().isIngestRunning()) { if (controller.isListeningEnabled()) { replaceNotification(centralStack, @@ -361,12 +384,17 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl } return; } + + // display a message about stuff still being in the queue if (controller.getDBTasksQueueSizeProperty().get() > 0) { replaceNotification(fullUIStack, new NoGroupsDialog(Bundle.ImageGalleryController_noGroupsDlg_msg3(), new ProgressIndicator())); return; } + + + // are there are files in the DB? try { if (controller.getDatabase().countAllFiles() <= 0) { // there are no files in db diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 56327fee07..ed9e1ebb54 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -151,8 +151,15 @@ public final class DrawableDB { private final Lock DBLock = rwLock.writeLock(); //using exclusing lock for all db ops for now + // caches to make inserts / updates faster private Cache groupCache = CacheBuilder.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build(); - + private boolean areCachesLoaded = false; // if true, the below caches contain valid data + private Set hasTagCache = new HashSet<>(); // contains obj id of files with tags + private Set hasHashCache = new HashSet<>(); // obj id of files with hash set hits + private Set hasExifCache = new HashSet<>(); // obj id of files with EXIF (make/model) + private int cacheBuildCount = 0; // number of tasks taht requested the caches be built + + static {//make sure sqlite driver is loaded // possibly redundant try { Class.forName("org.sqlite.JDBC"); @@ -773,16 +780,17 @@ public final class DrawableDB { insertOrUpdateFile(f, tr, updateFileStmt, caseDbTransaction); } - // WORK IN PROGRESS - Set hasTagCache = new HashSet<>(); - Set hasHashCache = new HashSet<>(); - Set hasExifCache = new HashSet<>(); - boolean areCachesLoaded = false; - int cacheUserCount = 0; + + // @@@ TODO: These caches shoudl be updated based on ingest events + /** + * Populate caches based on current state of Case DB + */ synchronized public void buildFileMetaDataCache() { - cacheUserCount++; + cacheBuildCount++; if (areCachesLoaded == true) return; + + // @@@ TODO Add better error handling try { // tag tags try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM content_tags")) { @@ -791,7 +799,6 @@ public final class DrawableDB { long id = rs.getLong("obj_id"); hasTagCache.add(id); } - } catch (SQLException ex) { Exceptions.printStackTrace(ex); } @@ -834,9 +841,12 @@ public final class DrawableDB { areCachesLoaded = true; } + /** + * Free the cached case DB data + */ synchronized public void freeFileMetaDataCache() { // dont' free these if there is another task still using them - if (--cacheUserCount > 0) + if (--cacheBuildCount > 0) return; areCachesLoaded = false; @@ -884,9 +894,11 @@ public final class DrawableDB { } stmt.setBoolean(9, f.isAnalyzed()); stmt.executeUpdate(); + // Update the list of file IDs in memory addImageFileToList(f.getId()); + // Update the hash set tables if ((!areCachesLoaded) || (hasHashCache.contains(f.getId()))) { try { for (String name : f.getHashSetNames()) { @@ -939,6 +951,7 @@ public final class DrawableDB { } } + // @@@ Consider storing more than ID so that we do not need to requery each file during commit tr.addUpdatedFile(f.getId()); } catch (SQLException | NullPointerException | TskCoreException ex) { @@ -1024,11 +1037,16 @@ public final class DrawableDB { return new DrawableTransaction(); } - public void commitTransaction(DrawableTransaction tr, Boolean notify) { + /** + * + * @param tr + * @param notifyGM If true, notify GroupManager about the changes. + */ + public void commitTransaction(DrawableTransaction tr, Boolean notifyGM) { if (tr.isClosed()) { throw new IllegalArgumentException("can't close already closed transaction"); } - tr.commit(notify); + tr.commit(notifyGM); } public void rollbackTransaction(DrawableTransaction tr) { @@ -1169,7 +1187,7 @@ public final class DrawableDB { * @param sortOrder Sort ascending or descending. * @param dataSource * - * @return + * @return Map of data source (or null of group by attribute ignores data sources) to list of unique group values * * @throws org.sleuthkit.datamodel.TskCoreException */ @@ -1617,14 +1635,19 @@ public final class DrawableDB { } } - synchronized private void commit(Boolean notify) { + /** + * Commit changes that happened during this transaction + * + * @param notifyGM If true, notify GroupManager about the changes. + */ + synchronized private void commit(Boolean notifyGM) { if (!closed) { try { con.commit(); // make sure we close before we update, bc they'll need locks close(); - if (notify) { + if (notifyGM) { if (groupManager != null) { groupManager.handleFileUpdate(updatedFiles); groupManager.handleFileRemoved(removedFiles); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java index 19eab6792a..ffcf307f21 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java @@ -93,7 +93,7 @@ public class HashSetManager { * * @param fileID the fileID to invalidate in the cache */ - public void invalidateHashSetsForFile(long fileID) { + public void invalidateHashSetsCacheForFile(long fileID) { hashSetCache.invalidate(fileID); } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 4309e82435..1140a54e8f 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -172,12 +172,15 @@ public class GroupManager { * a part of. */ @SuppressWarnings({"rawtypes", "unchecked"}) - synchronized public Set> getGroupKeysForFile(DrawableFile file) throws TskCoreException, TskDataException { + synchronized public Set> getGroupKeysForCurrentGroupBy(DrawableFile file) throws TskCoreException, TskDataException { Set> resultSet = new HashSet<>(); for (Comparable val : getGroupBy().getValue(file)) { if (getGroupBy() == DrawableAttribute.PATH) { - resultSet.add(new GroupKey(getGroupBy(), val, file.getDataSource())); + // verify this file is in a data source being displayed + if ((getDataSource() == null) || (file.getDataSource().equals(getDataSource()))) { + resultSet.add(new GroupKey(getGroupBy(), val, file.getDataSource())); + } } else if (getGroupBy() == DrawableAttribute.TAGS) { //don't show groups for the categories when grouped by tags. if (CategoryManager.isNotCategoryTagName((TagName) val)) { @@ -199,10 +202,10 @@ public class GroupManager { * @return A set of GroupKeys representing the group(s) the given file is a * part of */ - synchronized public Set> getGroupKeysForFileID(Long fileID) { + synchronized public Set> getGroupKeysForCurrentGroupBy(Long fileID) { try { DrawableFile file = getDrawableDB().getFileFromID(fileID); - return getGroupKeysForFile(file); + return getGroupKeysForCurrentGroupBy(file); } catch (TskCoreException | TskDataException ex) { logger.log(Level.SEVERE, "Failed to get group keys for file with ID " +fileID, ex); //NON-NLS @@ -434,10 +437,18 @@ public class GroupManager { return sortOrderProp.getReadOnlyProperty(); } + /** + * + * @return null if all data sources are being displayed + */ public synchronized DataSource getDataSource() { return dataSourceProp.get(); } + /** + * + * @param dataSource Data source to display or null to display all of them + */ synchronized void setDataSource(DataSource dataSource) { dataSourceProp.set(dataSource); } @@ -505,16 +516,28 @@ public class GroupManager { } } + /** + * Adds an analyzed file to a group and marks the group as analyzed if the entire group is + * now analyzed. + * + * @param group Group being added to (will be null if a group has not yet been created) + * @param groupKey Group type/value + * @param fileID + */ @SuppressWarnings("AssignmentToMethodParameter") synchronized private void addFileToGroup(DrawableGroup group, final GroupKey groupKey, final long fileID) { + + // NOTE: We assume that it has already been determined that GroupKey can be displayed based on Data Source filters if (group == null) { //if there wasn't already a group check if there should be one now + // path group, for example, only gets created when all files are analyzed group = popuplateIfAnalyzed(groupKey, null); } - if (group != null) { + else { //if there is aleady a group that was previously deemed fully analyzed, then add this newly analyzed file to it. group.addFile(fileID); } + // reset the seen status for the group markGroupSeen(group, false); } @@ -543,7 +566,7 @@ public class GroupManager { for (final long fileId : removedFileIDs) { //get grouping(s) this file would be in - Set> groupsForFile = getGroupKeysForFileID(fileId); + Set> groupsForFile = getGroupKeysForCurrentGroupBy(fileId); for (GroupKey gk : groupsForFile) { removeFromGroup(gk, fileId); @@ -563,13 +586,14 @@ public class GroupManager { * the problem is that as a new files are analyzed they might be in new * groups( if we are grouping by say make or model) -jm */ - for (long fileId : updatedFileIDs) { + for (long fileId : updatedFileIDs) { + // reset the hash cache + controller.getHashSetManager().invalidateHashSetsCacheForFile(fileId); - controller.getHashSetManager().invalidateHashSetsForFile(fileId); - - //get grouping(s) this file would be in - Set> groupsForFile = getGroupKeysForFileID(fileId); + // Update the current groups (if it is visible) + Set> groupsForFile = getGroupKeysForCurrentGroupBy(fileId); for (GroupKey gk : groupsForFile) { + // see if a group has been created yet for the key DrawableGroup g = getGroupForKey(gk); addFileToGroup(g, gk, fileId); } @@ -579,6 +603,10 @@ public class GroupManager { controller.getCategoryManager().fireChange(updatedFileIDs, null); } + /** + * If the group is analyzed (or other criteria based on grouping) and should be shown to the user, + * then add it to the appropriate data structures so that it can be viewed. + */ synchronized private DrawableGroup popuplateIfAnalyzed(GroupKey groupKey, ReGroupTask task) { /* * If this method call is part of a ReGroupTask and that task is @@ -588,44 +616,45 @@ public class GroupManager { * user picked a different group by attribute, while the current task * was still running) */ - if (isNull(task) || task.isCancelled() == false) { + if (isNull(task) == false && task.isCancelled() == true) { + return null; + } + + /* + * For attributes other than path we can't be sure a group is fully + * analyzed because we don't know all the files that will be a part + * of that group. just show them no matter what. + */ + if (groupKey.getAttribute() != DrawableAttribute.PATH + || getDrawableDB().isGroupAnalyzed(groupKey)) { + try { + Set fileIDs = getFileIDsInGroup(groupKey); + if (Objects.nonNull(fileIDs)) { - /* - * For attributes other than path we can't be sure a group is fully - * analyzed because we don't know all the files that will be a part - * of that group. just show them no matter what. - */ - if (groupKey.getAttribute() != DrawableAttribute.PATH - || getDrawableDB().isGroupAnalyzed(groupKey)) { - try { - Set fileIDs = getFileIDsInGroup(groupKey); - if (Objects.nonNull(fileIDs)) { + long examinerID = collaborativeModeProp.get() ? -1 : controller.getSleuthKitCase().getCurrentExaminer().getId(); + final boolean groupSeen = getDrawableDB().isGroupSeenByExaminer(groupKey, examinerID); + DrawableGroup group; - long examinerID = collaborativeModeProp.get() ? -1 : controller.getSleuthKitCase().getCurrentExaminer().getId(); - final boolean groupSeen = getDrawableDB().isGroupSeenByExaminer(groupKey, examinerID); - DrawableGroup group; - - if (groupMap.containsKey(groupKey)) { - group = groupMap.get(groupKey); - group.setFiles(fileIDs); - group.setSeen(groupSeen); - } else { - group = new DrawableGroup(groupKey, fileIDs, groupSeen); - controller.getCategoryManager().registerListener(group); - groupMap.put(groupKey, group); - } - - if (analyzedGroups.contains(group) == false) { - analyzedGroups.add(group); - sortAnalyzedGroups(); - } - updateUnSeenGroups(group); - - return group; + if (groupMap.containsKey(groupKey)) { + group = groupMap.get(groupKey); + group.setFiles(fileIDs); + group.setSeen(groupSeen); + } else { + group = new DrawableGroup(groupKey, fileIDs, groupSeen); + controller.getCategoryManager().registerListener(group); + groupMap.put(groupKey, group); } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "failed to get files for group: " + groupKey.getAttribute().attrName.toString() + " = " + groupKey.getValue(), ex); //NON-NLS + + if (analyzedGroups.contains(group) == false) { + analyzedGroups.add(group); + sortAnalyzedGroups(); + } + updateUnSeenGroups(group); + + return group; } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "failed to get files for group: " + groupKey.getAttribute().attrName.toString() + " = " + groupKey.getValue(), ex); //NON-NLS } } @@ -810,7 +839,7 @@ public class GroupManager { * * @param groupBy * - * @return + * @return map of data source (or null if group by attribute ignores data sources) to list of unique group values */ public Multimap findValuesForAttribute() { From bb84055f337676fc34e10a478e61ff0145531fce Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 26 Sep 2018 14:41:27 -0400 Subject: [PATCH 5/8] 1049: Image gallery rebuild does not complete for some images. - Only process 'real' files (not folders etc.) for Image gallery. --- .../autopsy/imagegallery/ImageGalleryController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 802db10a35..f485ae9cb9 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -666,8 +666,10 @@ public final class ImageGalleryController { break; } - processFile(f, drawableDbTransaction, caseDbTransaction); - + if (f.isFile()) { + processFile(f, drawableDbTransaction, caseDbTransaction); + } + workDone++; progressHandle.progress(f.getName(), workDone); updateProgress(workDone - 1 / (double) files.size()); From bc345144a36a973d0e9734514dc030deb2ff3299 Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 26 Sep 2018 15:48:27 -0400 Subject: [PATCH 6/8] 1049: process regular files only in Image gallery. - addressed review comment on previous commit. --- .../autopsy/imagegallery/ImageGalleryController.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index f485ae9cb9..53bbf62bdb 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -605,6 +605,7 @@ public final class ImageGalleryController { DRAWABLE_QUERY = DATASOURCE_CLAUSE + + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + " AND ( " + //grab files with supported extension FILE_EXTENSION_CLAUSE @@ -666,9 +667,7 @@ public final class ImageGalleryController { break; } - if (f.isFile()) { - processFile(f, drawableDbTransaction, caseDbTransaction); - } + processFile(f, drawableDbTransaction, caseDbTransaction); workDone++; progressHandle.progress(f.getName(), workDone); From 28be6a5ed605c041516850f0f96df75ceb2224aa Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Wed, 26 Sep 2018 23:44:17 -0400 Subject: [PATCH 7/8] Update caches based on events. Get DS Obj Id directly --- .../imagegallery/ImageGalleryModule.java | 101 ++++++++++++------ .../imagegallery/datamodel/DrawableDB.java | 56 +++++++--- .../imagegallery/datamodel/DrawableFile.java | 4 +- 3 files changed, 115 insertions(+), 46 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 90634093ee..6902dac224 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -27,6 +27,7 @@ import javafx.application.Platform; import javax.swing.JOptionPane; import javax.swing.SwingUtilities; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -39,9 +40,13 @@ import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManager.IngestJobEvent; +import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.DATA_ADDED; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.FILE_DONE; +import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -153,45 +158,72 @@ public class ImageGalleryModule { IngestManager.getInstance().removeIngestModuleEventListener(this); return; } - - if (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName()) != FILE_DONE) { - return; - } - // getOldValue has fileID getNewValue has Abstractfile - AbstractFile file = (AbstractFile) evt.getNewValue(); - if (false == file.isFile()) { - return; - } + /* only process individual files in realtime on the node that is * running the ingest. on a remote node, image files are processed * enblock when ingest is complete */ if (((AutopsyEvent) evt).getSourceType() != AutopsyEvent.SourceType.LOCAL) { return; } - + + // Bail out if the case is closed try { - ImageGalleryController con = getController(); - if (con.isListeningEnabled()) { - try { - if (isDrawableAndNotKnown(file)) { - //this file should be included and we don't already know about it from hash sets (NSRL) - con.queueDBTask(new ImageGalleryController.UpdateFileTask(file, controller.getDatabase())); - } else if (FileTypeUtils.getAllSupportedExtensions().contains(file.getNameExtension())) { - /* Doing this check results in fewer tasks queued - * up, and faster completion of db update. This file - * would have gotten scooped up in initial grab, but - * actually we don't need it */ - con.queueDBTask(new ImageGalleryController.RemoveFileTask(file, controller.getDatabase())); - } - - } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - logger.log(Level.SEVERE, "Unable to determine if file is drawable and not known. Not making any changes to DB", ex); //NON-NLS - MessageNotifyUtil.Notify.error("Image Gallery Error", - "Unable to determine if file is drawable and not known. Not making any changes to DB. See the logs for details."); - } + if (controller == null || Case.getCurrentCaseThrows() == null) { + return; } } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS + return; + } + + if (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName()) == FILE_DONE) { + + // getOldValue has fileID getNewValue has Abstractfile + AbstractFile file = (AbstractFile) evt.getNewValue(); + if (false == file.isFile()) { + return; + } + + try { + ImageGalleryController con = getController(); + if (con.isListeningEnabled()) { + try { + // Update the entry if it is a picture and not in NSRL + if (isDrawableAndNotKnown(file)) { + con.queueDBTask(new ImageGalleryController.UpdateFileTask(file, controller.getDatabase())); + } + // Remove it from the DB if it is no longer relevant, but had the correct extension + else if (FileTypeUtils.getAllSupportedExtensions().contains(file.getNameExtension())) { + /* Doing this check results in fewer tasks queued + * up, and faster completion of db update. This file + * would have gotten scooped up in initial grab, but + * actually we don't need it */ + con.queueDBTask(new ImageGalleryController.RemoveFileTask(file, controller.getDatabase())); + } + } catch (FileTypeDetector.FileTypeDetectorInitException ex) { + logger.log(Level.SEVERE, "Unable to determine if file is drawable and not known. Not making any changes to DB", ex); //NON-NLS + MessageNotifyUtil.Notify.error("Image Gallery Error", + "Unable to determine if file is drawable and not known. Not making any changes to DB. See the logs for details."); + } + } + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS + } + } + else if (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName()) == DATA_ADDED) { + ModuleDataEvent mde = (ModuleDataEvent)evt.getOldValue(); + + if (mde.getBlackboardArtifactType().getTypeID() == ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID()) { + DrawableDB drawableDB = controller.getDatabase(); + for (BlackboardArtifact art : mde.getArtifacts()) { + drawableDB.addExifCache(art.getObjectID()); + } + } + else if (mde.getBlackboardArtifactType().getTypeID() == ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID()) { + DrawableDB drawableDB = controller.getDatabase(); + for (BlackboardArtifact art : mde.getArtifacts()) { + drawableDB.addHashSetCache(art.getObjectID()); + } + } } } } @@ -251,7 +283,14 @@ public class ImageGalleryModule { break; case CONTENT_TAG_ADDED: final ContentTagAddedEvent tagAddedEvent = (ContentTagAddedEvent) evt; - if (con.getDatabase().isInDB(tagAddedEvent.getAddedTag().getContent().getId())) { + + long objId = tagAddedEvent.getAddedTag().getContent().getId(); + + // update the cache + DrawableDB drawableDB = controller.getDatabase(); + drawableDB.addTagCache(objId); + + if (con.getDatabase().isInDB(objId)) { con.getTagsManager().fireTagAddedEvent(tagAddedEvent); } break; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index ed9e1ebb54..caf22e5799 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -781,7 +781,6 @@ public final class DrawableDB { } - // @@@ TODO: These caches shoudl be updated based on ingest events /** * Populate caches based on current state of Case DB */ @@ -789,10 +788,9 @@ public final class DrawableDB { cacheBuildCount++; if (areCachesLoaded == true) return; - - // @@@ TODO Add better error handling + try { - // tag tags + // get tags try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM content_tags")) { ResultSet rs = dbQuery.getResultSet(); while (rs.next()) { @@ -800,10 +798,10 @@ public final class DrawableDB { hasTagCache.add(id); } } catch (SQLException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.SEVERE, "Error getting tags from DB", ex); //NON-NLS } } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.SEVERE, "Error executing query to get tags", ex); //NON-NLS } try { @@ -816,10 +814,10 @@ public final class DrawableDB { } } catch (SQLException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.SEVERE, "Error getting hashsets from DB", ex); //NON-NLS } } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.SEVERE, "Error executing query to get hashsets", ex); //NON-NLS } try { @@ -832,15 +830,48 @@ public final class DrawableDB { } } catch (SQLException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.SEVERE, "Error getting EXIF from DB", ex); //NON-NLS } } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.SEVERE, "Error executing query to get EXIF", ex); //NON-NLS } areCachesLoaded = true; } + /** + * Add a file to cache of files that have EXIF data + * @param objectID ObjId of file with EXIF + */ + public void addExifCache(long objectID) { + // bail out if we are not maintaining caches + if (cacheBuildCount == 0) + return; + hasExifCache.add(objectID); + } + + /** + * Add a file to cache of files that have hash set hits + * @param objectID ObjId of file with hash set + */ + public void addHashSetCache(long objectID) { + // bail out if we are not maintaining caches + if (cacheBuildCount == 0) + return; + hasHashCache.add(objectID); + } + + /** + * Add a file to cache of files that have tags + * @param objectID ObjId of file with tags + */ + public void addTagCache(long objectID) { + // bail out if we are not maintaining caches + if (cacheBuildCount == 0) + return; + hasTagCache.add(objectID); + } + /** * Free the cached case DB data */ @@ -866,7 +897,7 @@ public final class DrawableDB { * * @param f The file to insert. * @param tr a transaction to use, must not be null - * @param stmt the statement that does the actull inserting + * @param stmt the statement that does the actual inserting */ private void insertOrUpdateFile(DrawableFile f, @Nonnull DrawableTransaction tr, @Nonnull PreparedStatement stmt, @Nonnull CaseDbTransaction caseDbTransaction) { @@ -878,8 +909,7 @@ public final class DrawableDB { try { // "INSERT OR IGNORE/ INTO drawable_files (obj_id, data_source_obj_id, path, name, created_time, modified_time, make, model, analyzed)" stmt.setLong(1, f.getId()); - // @@@ Should be able to get ID directly from abstract file... - stmt.setLong(2, f.getAbstractFile().getDataSource().getId()); + stmt.setLong(2, f.getAbstractFile().getDataSourceObjectId()); stmt.setString(3, f.getDrawablePath()); stmt.setString(4, f.getName()); stmt.setLong(5, f.getCrtime()); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableFile.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableFile.java index ae3b16ae5c..4b777b387d 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableFile.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableFile.java @@ -63,8 +63,8 @@ public abstract class DrawableFile { private static final Logger LOGGER = Logger.getLogger(DrawableFile.class.getName()); - public static DrawableFile create(AbstractFile abstractFileById, boolean analyzed) { - return create(abstractFileById, analyzed, FileTypeUtils.hasVideoMIMEType(abstractFileById)); + public static DrawableFile create(AbstractFile abstractFile, boolean analyzed) { + return create(abstractFile, analyzed, FileTypeUtils.hasVideoMIMEType(abstractFile)); } /** From ae2464a19a79030803a1f3177951f78160772e00 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 27 Sep 2018 12:28:38 -0400 Subject: [PATCH 8/8] Added syncronization to the caches --- .../imagegallery/datamodel/DrawableDB.java | 181 ++++++++++-------- 1 file changed, 100 insertions(+), 81 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index caf22e5799..f7a9c2b768 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -54,7 +54,6 @@ import javax.annotation.concurrent.GuardedBy; import javax.swing.SortOrder; import static org.apache.commons.lang3.ObjectUtils.notEqual; import org.apache.commons.lang3.StringUtils; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.DhsImageCategory; @@ -153,6 +152,7 @@ public final class DrawableDB { // caches to make inserts / updates faster private Cache groupCache = CacheBuilder.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build(); + private final Object cacheLock = new Object(); // protects access to the below cache-related objects private boolean areCachesLoaded = false; // if true, the below caches contain valid data private Set hasTagCache = new HashSet<>(); // contains obj id of files with tags private Set hasHashCache = new HashSet<>(); // obj id of files with hash set hits @@ -784,59 +784,62 @@ public final class DrawableDB { /** * Populate caches based on current state of Case DB */ - synchronized public void buildFileMetaDataCache() { - cacheBuildCount++; - if (areCachesLoaded == true) - return; + public void buildFileMetaDataCache() { + + synchronized (cacheLock) { + cacheBuildCount++; + if (areCachesLoaded == true) + return; - try { - // get tags - try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM content_tags")) { - ResultSet rs = dbQuery.getResultSet(); - while (rs.next()) { - long id = rs.getLong("obj_id"); - hasTagCache.add(id); + try { + // get tags + try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM content_tags")) { + ResultSet rs = dbQuery.getResultSet(); + while (rs.next()) { + long id = rs.getLong("obj_id"); + hasTagCache.add(id); + } + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Error getting tags from DB", ex); //NON-NLS } - } catch (SQLException ex) { - logger.log(Level.SEVERE, "Error getting tags from DB", ex); //NON-NLS + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error executing query to get tags", ex); //NON-NLS } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error executing query to get tags", ex); //NON-NLS - } - - try { - // hash sets - try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM blackboard_artifacts WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID())) { - ResultSet rs = dbQuery.getResultSet(); - while (rs.next()) { - long id = rs.getLong("obj_id"); - hasHashCache.add(id); + + try { + // hash sets + try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM blackboard_artifacts WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID())) { + ResultSet rs = dbQuery.getResultSet(); + while (rs.next()) { + long id = rs.getLong("obj_id"); + hasHashCache.add(id); + } + + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Error getting hashsets from DB", ex); //NON-NLS } - - } catch (SQLException ex) { - logger.log(Level.SEVERE, "Error getting hashsets from DB", ex); //NON-NLS + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error executing query to get hashsets", ex); //NON-NLS } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error executing query to get hashsets", ex); //NON-NLS - } - - try { - // EXIF - try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM blackboard_artifacts WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID())) { - ResultSet rs = dbQuery.getResultSet(); - while (rs.next()) { - long id = rs.getLong("obj_id"); - hasExifCache.add(id); + + try { + // EXIF + try (SleuthkitCase.CaseDbQuery dbQuery = tskCase.executeQuery("SELECT obj_id FROM blackboard_artifacts WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID())) { + ResultSet rs = dbQuery.getResultSet(); + while (rs.next()) { + long id = rs.getLong("obj_id"); + hasExifCache.add(id); + } + + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Error getting EXIF from DB", ex); //NON-NLS } - - } catch (SQLException ex) { - logger.log(Level.SEVERE, "Error getting EXIF from DB", ex); //NON-NLS + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error executing query to get EXIF", ex); //NON-NLS } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error executing query to get EXIF", ex); //NON-NLS + + areCachesLoaded = true; } - - areCachesLoaded = true; } /** @@ -844,10 +847,12 @@ public final class DrawableDB { * @param objectID ObjId of file with EXIF */ public void addExifCache(long objectID) { - // bail out if we are not maintaining caches - if (cacheBuildCount == 0) - return; - hasExifCache.add(objectID); + synchronized (cacheLock) { + // bail out if we are not maintaining caches + if (cacheBuildCount == 0) + return; + hasExifCache.add(objectID); + } } /** @@ -855,10 +860,12 @@ public final class DrawableDB { * @param objectID ObjId of file with hash set */ public void addHashSetCache(long objectID) { - // bail out if we are not maintaining caches - if (cacheBuildCount == 0) - return; - hasHashCache.add(objectID); + synchronized (cacheLock) { + // bail out if we are not maintaining caches + if (cacheBuildCount == 0) + return; + hasHashCache.add(objectID); + } } /** @@ -866,24 +873,28 @@ public final class DrawableDB { * @param objectID ObjId of file with tags */ public void addTagCache(long objectID) { - // bail out if we are not maintaining caches - if (cacheBuildCount == 0) - return; - hasTagCache.add(objectID); + synchronized (cacheLock) { + // bail out if we are not maintaining caches + if (cacheBuildCount == 0) + return; + hasTagCache.add(objectID); + } } /** * Free the cached case DB data */ - synchronized public void freeFileMetaDataCache() { - // dont' free these if there is another task still using them - if (--cacheBuildCount > 0) - return; - - areCachesLoaded = false; - hasTagCache.clear(); - hasHashCache.clear(); - hasExifCache.clear(); + public void freeFileMetaDataCache() { + synchronized (cacheLock) { + // dont' free these if there is another task still using them + if (--cacheBuildCount > 0) + return; + + areCachesLoaded = false; + hasTagCache.clear(); + hasHashCache.clear(); + hasExifCache.clear(); + } } /** @@ -904,6 +915,18 @@ public final class DrawableDB { if (tr.isClosed()) { throw new IllegalArgumentException("can't update database with closed transaction"); } + + // get data from caches. Default to true and force the DB lookup if we don't have caches + boolean hasExif = true; + boolean hasHashSet = true; + boolean hasTag = true; + synchronized (cacheLock) { + if (areCachesLoaded) { + hasExif = hasExifCache.contains(f.getId()); + hasHashSet = hasHashCache.contains(f.getId()); + hasTag = hasTagCache.contains(f.getId()); + } + } dbWriteLock(); try { @@ -914,13 +937,12 @@ public final class DrawableDB { stmt.setString(4, f.getName()); stmt.setLong(5, f.getCrtime()); stmt.setLong(6, f.getMtime()); - if ((areCachesLoaded) && (hasExifCache.contains(f.getId()) == false)) { - stmt.setString(7, ""); - stmt.setString(8, ""); - } - else { + if (hasExif) { stmt.setString(7, f.getMake()); stmt.setString(8, f.getModel()); + } else { + stmt.setString(7, ""); + stmt.setString(8, ""); } stmt.setBoolean(9, f.isAnalyzed()); stmt.executeUpdate(); @@ -929,7 +951,7 @@ public final class DrawableDB { addImageFileToList(f.getId()); // Update the hash set tables - if ((!areCachesLoaded) || (hasHashCache.contains(f.getId()))) { + if (hasHashSet) { try { for (String name : f.getHashSetNames()) { @@ -958,15 +980,12 @@ public final class DrawableDB { //and update all groups this file is in for (DrawableAttribute attr : DrawableAttribute.getGroupableAttrs()) { - if (attr == DrawableAttribute.TAGS) { - if ((!areCachesLoaded) || (hasTagCache.contains(f.getId()) == false)) { - continue; - } + // skip attributes that we do not have data for + if ((attr == DrawableAttribute.TAGS) && (hasTag == false)) { + continue; } - else if (attr == DrawableAttribute.MAKE || attr == DrawableAttribute.MODEL) { - if ((!areCachesLoaded) || (hasExifCache.contains(f.getId()) == false)) { - continue; - } + else if ((attr == DrawableAttribute.MAKE || attr == DrawableAttribute.MODEL) && (hasExif == false)) { + continue; } Collection> vals = attr.getValue(f); for (Comparable val : vals) {