From cc355d99b6ea6c691965c23c62880f5565b2cb17 Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Mon, 19 Mar 2018 14:00:09 -0600 Subject: [PATCH] files deduped in only 2 sql queries - no roundtrips business logic encapsulated outside of swingworker --- .../commonfilesearch/CommonFilesMetaData.java | 65 +++++++++++++------ .../commonfilesearch/CommonFilesPanel.java | 38 +++++------ 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java index c59bf793ec..394841037f 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java @@ -19,6 +19,7 @@ */ package org.sleuthkit.autopsy.commonfilesearch; +import java.sql.ResultSet; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -27,8 +28,10 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** @@ -40,13 +43,41 @@ public class CommonFilesMetaData { private final List dedupedFiles; private final Map instanceCountMap; - private final Map dataSourceMap; - private final Map dataSourcesMap; + private final Map dataSourceIdToNameMap; + private final Map md5ToDataSourcesStringMap; + + private SleuthkitCase sleuthkitCase; + CommonFilesMetaData() { + dedupedFiles = new ArrayList<>(); + instanceCountMap = new HashMap<>(); + md5ToDataSourcesStringMap = new HashMap<>(); + dataSourceIdToNameMap = new HashMap<>(); + + this.sleuthkitCase = Case.getOpenCase().getSleuthkitCase(); + + this.loadDataSourcesMap(); + + this.collateFiles(); + } + + private void loadDataSourcesMap(){ + SleuthkitCase.CaseDbQuery query = this.sleuthkitCase.executeQuery("select obj_id, name from tsk_files where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))"); + + ResultSet resultSet = query.getResultSet(); + + while(resultSet.next()){ + Long objectId = resultSet.getLong(1); + String dataSourceName = resultSet.getString(2); + this.dataSourceIdToNameMap.put(objectId, dataSourceName); + } + } + CommonFilesMetaData(List theDedupedFiles, Map theDataSourceMap, Map theInstanceCountMap) { dedupedFiles = theDedupedFiles; instanceCountMap = theInstanceCountMap; - dataSourcesMap = theDataSourceMap; + md5ToDataSourcesStringMap = theDataSourceMap; + dataSourceIdToNameMap = new HashMap<>(); } public List getFilesList() { @@ -58,7 +89,7 @@ public class CommonFilesMetaData { } public Map getDataSourceMap() { - return Collections.unmodifiableMap(dataSourcesMap); + return Collections.unmodifiableMap(md5ToDataSourcesStringMap); } /** @@ -72,10 +103,9 @@ public class CommonFilesMetaData { * @return object with deduped file list and maps of files to data sources * and number instances */ - static CommonFilesMetaData CollateFiles(List files) { - List deDupedFiles = new ArrayList<>(); - java.util.Map dataSourceMap = new HashMap<>(); - java.util.Map instanceCountMap = new HashMap<>(); + private void collateFiles() { + + List files = this.sleuthkitCase.findAllFilesWhere("md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL) GROUP BY md5 HAVING COUNT(*) > 1) order by md5"); AbstractFile previousFile = null; String previousMd5 = ""; @@ -91,9 +121,9 @@ public class CommonFilesMetaData { addDataSource(dataSources, file); } else { if (previousFile != null) { - deDupedFiles.add(previousFile); - instanceCountMap.put(previousMd5, instanceCount); - dataSourceMap.put(previousMd5, String.join(", ", dataSources)); + this.dedupedFiles.add(previousFile); + this.instanceCountMap.put(previousMd5, instanceCount); + this.md5ToDataSourcesStringMap.put(previousMd5, String.join(", ", dataSources)); } previousFile = file; previousMd5 = currentMd5; @@ -102,16 +132,11 @@ public class CommonFilesMetaData { addDataSource(dataSources, file); } } - CommonFilesMetaData data = new CommonFilesMetaData(deDupedFiles, dataSourceMap, instanceCountMap); - return data; } - private static void addDataSource(Set dataSources, AbstractFile file) { - try { - dataSources.add(file.getDataSource().getName()); - } catch (TskCoreException ex) { - LOGGER.log(Level.WARNING, String.format("Unable to determine data source for file with ID: {0}.", file.getId()), ex); - dataSources.add("Unknown Source"); - } + private void addDataSource(Set dataSources, AbstractFile file) { + long datasourceId = file.getDataSourceObjectId(); + String dataSourceName = this.dataSourceIdToNameMap.get(datasourceId); + dataSources.add(dataSourceName); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java index fb9493203e..65ad309d79 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java @@ -75,11 +75,11 @@ public final class CommonFilesPanel extends javax.swing.JPanel { String title = Bundle.CommonFilesPanel_search_results_title(); String pathText = Bundle.CommonFilesPanel_search_results_pathText(); - new SwingWorker, Void>() { + new SwingWorker() { @Override @SuppressWarnings("FinallyDiscardsException") - protected List doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException { //return type should be CommonFilesMetaData - done will be adjusted accordingly + protected CommonFilesMetaData doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException { //return type should be CommonFilesMetaData - done will be adjusted accordingly //contents of this whole function should be wrapped in a business logic class for the sake of testing @@ -93,22 +93,22 @@ public final class CommonFilesPanel extends javax.swing.JPanel { Use file.getDataSourceID() to get datasourceid and map it to the appropriate row from above */ - Case currentCase = Case.getOpenCase(); - SleuthkitCase tskDb = currentCase.getSleuthkitCase(); +// Case currentCase = Case.getOpenCase(); +// SleuthkitCase tskDb = currentCase.getSleuthkitCase(); +// +// CaseDbQuery query = tskDb.executeQuery("select obj_id, name from tsk_files where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))"); +// +// ResultSet resultSet = query.getResultSet(); +// +// Map dataSourceMap = new HashMap<>(); +// +// while(resultSet.next()){ +// Long objectId = resultSet.getLong(1); +// String dataSourceName = resultSet.getString(2); +// dataSourceMap.put(objectId, dataSourceName); +// } - CaseDbQuery query = tskDb.executeQuery("select obj_id, name from tsk_files where obj_id in (SELECT obj_id FROM tsk_objects WHERE obj_id in (select obj_id from data_source_info))"); - - ResultSet resultSet = query.getResultSet(); - - Map dataSourceMap = new HashMap<>(); - - while(resultSet.next()){ - Long objectId = resultSet.getLong(0); - String dataSourceName = resultSet.getString(1); - dataSourceMap.put(objectId, dataSourceName); - } - - return tskDb.findAllFilesWhere("md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL) GROUP BY md5 HAVING COUNT(*) > 1) order by md5"); + return new CommonFilesMetaData(); } @Override @@ -116,9 +116,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel { try { super.done(); - List contentList = get(); // - //// To background thread (doInBackground) - CommonFilesMetaData metadata = CommonFilesMetaData.CollateFiles(contentList); // + CommonFilesMetaData metadata = get(); CommonFilesSearchNode contentFilesNode = new CommonFilesSearchNode(metadata);