From 2afd45d861823bd8ca2bf0857c8eda32e2ea882a Mon Sep 17 00:00:00 2001 From: millmanorama Date: Tue, 11 Jul 2017 22:32:24 +0200 Subject: [PATCH] use a single query to get all mime type counts. --- .../autopsy/datamodel/FileTypes.java | 7 +- .../datamodel/FileTypesByExtension.java | 5 +- .../datamodel/FileTypesByMimeType.java | 77 ++++++++----------- 3 files changed, 40 insertions(+), 49 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index 956708b66b..6b8e502efc 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -96,7 +96,7 @@ public final class FileTypes implements AutopsyVisitableItem { super(new RootContentChildren(Arrays.asList( new FileTypesByExtension(FileTypes.this), new FileTypesByMimeType(FileTypes.this))), - Lookups.singleton(NAME)); + Lookups.singleton(NAME)); setName(NAME); setDisplayName(NAME); this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/file_types.png"); @@ -205,7 +205,7 @@ public final class FileTypes implements AutopsyVisitableItem { abstract String getDisplayNameBase(); - abstract String getQuery(); + abstract long calculateItems() throws Exception; /** * Updates the display name of the mediaSubTypeNode to include the count @@ -219,7 +219,7 @@ public final class FileTypes implements AutopsyVisitableItem { new SwingWorker() { @Override protected Long doInBackground() throws Exception { - return skCase.countFilesWhere(getQuery()); + return calculateItems(); } @Override @@ -232,6 +232,7 @@ public final class FileTypes implements AutopsyVisitableItem { logger.log(Level.WARNING, "Failed to get count of files for " + getDisplayNameBase(), ex); } } + }.execute(); } else { setDisplayName(getDisplayNameBase() + ((childCount < 0) ? "" diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java index b3d2020322..57c0663e4a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java @@ -325,10 +325,9 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { } @Override - String getQuery() { - return createQuery(filter); + long calculateItems() throws Exception { + return skCase.countFilesWhere(createQuery(filter)); } - } private static String createQuery(FileTypesByExtension.SearchFilterInterface filter) { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java index 01ccc24267..7731606da4 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java @@ -26,10 +26,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Observable; import java.util.Observer; import java.util.logging.Level; -import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; @@ -61,7 +61,7 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi * which exist in the database. This hashmap will store them with the media * type as the key and a list of media subtypes as the value. */ - private final HashMap> existingMimeTypes = new HashMap<>(); + private final HashMap> existingMimeTypes = new HashMap<>(); private static final Logger LOGGER = Logger.getLogger(FileTypesByMimeType.class.getName()); private final FileTypes typesRoot; @@ -95,35 +95,35 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi * files in it, and populate the hashmap with those results. */ private void populateHashMap() { - StringBuilder allDistinctMimeTypesQuery = new StringBuilder(); - allDistinctMimeTypesQuery.append("SELECT DISTINCT mime_type from tsk_files where mime_type IS NOT null"); //NON-NLS - allDistinctMimeTypesQuery.append(" AND dir_type = ").append(TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue()); //NON-NLS - allDistinctMimeTypesQuery.append(" AND (type IN (").append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.ordinal()).append(","); //NON-NLS - allDistinctMimeTypesQuery.append(TskData.TSK_DB_FILES_TYPE_ENUM.CARVED.ordinal()).append(","); - allDistinctMimeTypesQuery.append(TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED.ordinal()).append(","); - if (!UserPreferences.hideSlackFilesInViewsTree()) { - allDistinctMimeTypesQuery.append(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK.ordinal()).append(","); - } - allDistinctMimeTypesQuery.append(TskData.TSK_DB_FILES_TYPE_ENUM.LOCAL.ordinal()).append("))"); + String query = "SELECT mime_type,count(*) as count from tsk_files " + + " where mime_type IS NOT null " + + " AND dir_type = " + TskData.TSK_FS_NAME_TYPE_ENUM.REG.getValue() + + " AND (type IN (" + + TskData.TSK_DB_FILES_TYPE_ENUM.FS.ordinal() + "," + + TskData.TSK_DB_FILES_TYPE_ENUM.CARVED.ordinal() + "," + + TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED.ordinal() + "," + + TskData.TSK_DB_FILES_TYPE_ENUM.LOCAL.ordinal() + + ((UserPreferences.hideSlackFilesInViewsTree()) ? "" + : ("," + TskData.TSK_DB_FILES_TYPE_ENUM.SLACK.ordinal())) + + "))" + " GROUP BY mime_type"; synchronized (existingMimeTypes) { existingMimeTypes.clear(); if (skCase == null) { return; } - try (SleuthkitCase.CaseDbQuery dbQuery = skCase.executeQuery(allDistinctMimeTypesQuery.toString())) { + try (SleuthkitCase.CaseDbQuery dbQuery = skCase.executeQuery(query)) { ResultSet resultSet = dbQuery.getResultSet(); while (resultSet.next()) { final String mime_type = resultSet.getString("mime_type"); //NON-NLS if (!mime_type.isEmpty()) { - String mimeType[] = mime_type.split("/"); //if the mime_type contained multiple slashes then everything after the first slash will become the subtype - final String mimeMediaSubType = StringUtils.join(ArrayUtils.subarray(mimeType, 1, mimeType.length), "/"); - if (mimeType.length > 1 && !mimeType[0].isEmpty() && !mimeMediaSubType.isEmpty()) { - if (!existingMimeTypes.containsKey(mimeType[0])) { - existingMimeTypes.put(mimeType[0], new ArrayList<>()); - } - existingMimeTypes.get(mimeType[0]).add(mimeMediaSubType); + final String mediaType = StringUtils.substringBefore(mime_type, "/"); + final String subType = StringUtils.removeStart(mime_type, mediaType + "/"); + if (!mediaType.isEmpty() && !subType.isEmpty()) { + final long count = resultSet.getLong("count"); + existingMimeTypes.computeIfAbsent(mediaType, t -> new HashMap<>()) + .put(subType, count); } } } @@ -133,11 +133,10 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi } setChanged(); - notifyObservers(); } - FileTypesByMimeType( FileTypes typesRoot) { + FileTypesByMimeType(FileTypes typesRoot) { this.skCase = typesRoot.getSleuthkitCase(); this.typesRoot = typesRoot; this.pcl = (PropertyChangeEvent evt) -> { @@ -228,7 +227,9 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi } boolean isEmpty() { - return existingMimeTypes.isEmpty(); + synchronized (existingMimeTypes) { + return existingMimeTypes.isEmpty(); + } } } @@ -310,7 +311,7 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi @Override protected boolean createKeys(List mediaTypeNodes) { - mediaTypeNodes.addAll(existingMimeTypes.get(mediaType)); + mediaTypeNodes.addAll(existingMimeTypes.get(mediaType).keySet()); return true; } @@ -333,7 +334,7 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi private final String subType; private MediaSubTypeNode(String mimeType) { - super(typesRoot,Children.create(new MediaSubTypeNodeChildren(mimeType), true)); + super(typesRoot, Children.create(new MediaSubTypeNodeChildren(mimeType), true)); this.mimeType = mimeType; this.subType = StringUtils.substringAfter(mimeType, "/"); super.setName(mimeType); @@ -374,25 +375,15 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi return subType; } + /** + * Get children count without actually loading all nodes + * + * @return count(*) - the number of items that will be shown in this + * items Directory Listing + */ @Override - String getQuery() { - return createQuery(mimeType); - } - - } - - /** - * Get children count without actually loading all nodes - * - * @return count(*) - the number of items that will be shown in this items - * Directory Listing - */ - static private long calculateItems(SleuthkitCase sleuthkitCase, String mime_type) { - try { - return sleuthkitCase.countFilesWhere(createQuery(mime_type)); - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error getting file search view count", ex); //NON-NLS - return 0; + long calculateItems() { + return existingMimeTypes.get(StringUtils.substringBefore(mimeType, "/")).get(subType); } }