From d2ae65ece97f166d44ae75f58a838ab0cb8178ac Mon Sep 17 00:00:00 2001 From: Brian Sweeney Date: Tue, 17 Apr 2018 06:15:48 -0600 Subject: [PATCH] reimplemented algorithm so that parents and children are selected separately --- .../commonfilesearch/CommonFilesMetaData.java | 42 +++++----- .../CommonFilesMetaDataBuilder.java | 78 +++---------------- .../commonfilesearch/CommonFilesNode.java | 22 +++--- .../commonfilesearch/CommonFilesPanel.java | 18 ++--- .../FileInstanceMetaData.java | 48 ++++++++++++ .../autopsy/commonfilesearch/Md5MetaData.java | 59 ++++++++++++++ .../sleuthkit/autopsy/datamodel/Md5Node.java | 72 ++++++++++------- 7 files changed, 201 insertions(+), 138 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetaData.java create mode 100644 Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5MetaData.java diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java index 121ae4a4e5..fa93a04d37 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java @@ -19,44 +19,42 @@ */ package org.sleuthkit.autopsy.commonfilesearch; -import java.sql.SQLException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.datamodel.TskCoreException; /** * Utility and wrapper model around data required for Common Files Search results. * Subclass this to implement different selections of files from the case. */ public class CommonFilesMetaData { - - private final String parentMd5; - private final List children; - private final String dataSources; + + private final Map metadata; private final Map dataSourceIdToNameMap; - CommonFilesMetaData(String md5, List childNodes, String dataSourcesString, Map dataSourcesMap) throws TskCoreException, SQLException, NoCurrentCaseException { - parentMd5 = md5; - children = childNodes; - dataSources = dataSourcesString; - dataSourceIdToNameMap = dataSourcesMap; - } - - public String getMd5() { - return parentMd5; + CommonFilesMetaData(Map metadata, Map dataSourcesMap) { + this.metadata = metadata; + this.dataSourceIdToNameMap = dataSourcesMap; } - public List getChildren() { - return Collections.unmodifiableList(this.children); + public Md5MetaData getMetaDataForMd5(String md5){ + return this.metadata.get(md5); } - + + public Map getMataData(){ + return Collections.unmodifiableMap(this.metadata); + } + public Map getDataSourceIdToNameMap() { - return Collections.unmodifiableMap(dataSourceIdToNameMap); + return Collections.unmodifiableMap(this.dataSourceIdToNameMap); } - public String getDataSources() { - return dataSources; + int size() { + int count = 0; + for(Md5MetaData data : this.metadata.values()){ + count += data.size(); + } + return count; } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java index e9198a1b24..6302dec626 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java @@ -51,39 +51,6 @@ abstract class CommonFilesMetaDataBuilder { dataSourceIdToNameMap = dataSourceIdMap; } - private void addDataSource(Set dataSources, AbstractFile file, Map dataSourceIdToNameMap) { - long datasourceId = file.getDataSourceObjectId(); - String dataSourceName = dataSourceIdToNameMap.get(datasourceId); - dataSources.add(dataSourceName); - } - - /** - * Sorts files in selection into a parent/child hierarchy where actual files - * are nested beneath a parent node which represents the common match. - * - * @return returns a reference to itself for ease of use. - * @throws TskCoreException - */ - List collateFiles() throws TskCoreException, SQLException { - List metaDataModels = new ArrayList<>(); - Map> md5ToDataSourcesStringMap = new HashMap<>(); - - try { - List files = findCommonFiles(); - - Map> parentNodes = new HashMap<>(); - - collateParentChildRelationships(files, parentNodes, md5ToDataSourcesStringMap); - for (String key : parentNodes.keySet()) { - metaDataModels.add(new CommonFilesMetaData(key, parentNodes.get(key), String.join(", ", md5ToDataSourcesStringMap.get(key)), dataSourceIdToNameMap)); - } - } catch (NoCurrentCaseException ex) { - Exceptions.printStackTrace(ex); - } - - return metaDataModels; - } - protected static String SELECT_PREFIX = "SELECT obj_id, md5, data_source_obj_id from tsk_files where"; /** @@ -97,33 +64,9 @@ abstract class CommonFilesMetaDataBuilder { */ protected abstract String buildSqlSelectStatement(); - private void collateParentChildRelationships(List files, Map> parentNodes, Map> md5ToDataSourcesStringMap) { - for (AbstractFile file : files) { - - String currentMd5 = file.getMd5Hash(); - if((currentMd5 == null) || (HashUtility.isNoDataMd5(currentMd5))) { - continue; - } - if (parentNodes.containsKey(currentMd5)) { - parentNodes.get(currentMd5).add(file); - Set currentDataSources = md5ToDataSourcesStringMap.get(currentMd5); - addDataSource(currentDataSources, file, dataSourceIdToNameMap); - md5ToDataSourcesStringMap.put(currentMd5, currentDataSources); - - } else { - List children = new ArrayList<>(); - Set dataSources = new HashSet<>(); - children.add(file); - parentNodes.put(currentMd5, children); - addDataSource(dataSources, file, dataSourceIdToNameMap); - md5ToDataSourcesStringMap.put(currentMd5, dataSources); - } - } - } - - private Map> findCommonFiles() throws TskCoreException, NoCurrentCaseException, SQLException { + public Map findCommonFiles() throws TskCoreException, NoCurrentCaseException, SQLException { - Map> md5ToObjIdMap = new HashMap<>(); + Map commonFiles = new HashMap<>(); SleuthkitCase sleuthkitCase = Case.getOpenCase().getSleuthkitCase(); String selectStatement = this.buildSqlSelectStatement(); @@ -131,19 +74,22 @@ abstract class CommonFilesMetaDataBuilder { try (CaseDbQuery query = sleuthkitCase.executeQuery(selectStatement)){ ResultSet resultSet = query.getResultSet(); while(resultSet.next()){ - String md5 = resultSet.getString(1); - Long objectId = resultSet.getLong(2); + Long objectId = resultSet.getLong(1); + String md5 = resultSet.getString(2); Long dataSourceId = resultSet.getLong(3); + String dataSource = this.dataSourceIdToNameMap.get(dataSourceId); - if(md5ToObjIdMap.containsKey(md5)){ - md5ToObjIdMap.get(md5).add(objectId); + if(commonFiles.containsKey(md5)){ + commonFiles.get(md5).getMetaData().add(new FileInstanceMetaData(objectId, dataSource, dataSourceId)); } else { - List objectIds = new ArrayList<>(); - md5ToObjIdMap.put(md5, objectIds); + List fileInstances = new ArrayList<>(); + fileInstances.add(new FileInstanceMetaData(objectId, dataSource, dataSourceId)); + Md5MetaData md5s = new Md5MetaData(md5, fileInstances); + commonFiles.put(md5, md5s); } } } - return md5ToObjIdMap; + return commonFiles; } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesNode.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesNode.java index 46ff8f8ec9..7d6ce4ab46 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesNode.java @@ -34,7 +34,7 @@ import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor; */ final public class CommonFilesNode extends DisplayableItemNode { - CommonFilesNode(List metaDataList) { + CommonFilesNode(CommonFilesMetaData metaDataList) { super(Children.create(new Md5NodeFactory(metaDataList), true), Lookups.singleton(CommonFilesNode.class)); } @@ -64,31 +64,31 @@ final public class CommonFilesNode extends DisplayableItemNode { * ChildFactory which builds CommonFileParentNodes from the * CommonFilesMetaaData models. */ - static class Md5NodeFactory extends ChildFactory { + static class Md5NodeFactory extends ChildFactory { /** * List of models, each of which is a parent node matching a single md5, * containing children FileNodes. */ - private List metaDataList; + private CommonFilesMetaData metadata; - Md5NodeFactory(List theMetaDataList) { - this.metaDataList = theMetaDataList; + Md5NodeFactory(CommonFilesMetaData theMetaDataList) { + this.metadata = theMetaDataList; } protected void removeNotify() { - metaDataList = null; + metadata = null; } - + @Override - protected Node createNodeForKey(CommonFilesMetaData metaData) { - + protected Node createNodeForKey(String md5){ + Md5MetaData metaData = this.metadata.getMetaDataForMd5(md5); return new Md5Node(metaData); } @Override - protected boolean createKeys(List toPopulate) { - toPopulate.addAll(metaDataList); + protected boolean createKeys(List list) { + list.addAll(this.metadata.getMataData().keySet()); return true; } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java index 622bdc5290..ceca2ca1dd 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java @@ -213,7 +213,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel { String pathText = Bundle.CommonFilesPanel_search_results_pathText(); - new SwingWorker, Void>() { + new SwingWorker() { private String tabTitle; @@ -243,7 +243,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel { @Override @SuppressWarnings({"BoxedValueEquality", "NumberEquality"}) - protected List doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException { + protected CommonFilesMetaData doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException { Long dataSourceId = determineDataSourceId(); CommonFilesMetaDataBuilder builder; @@ -257,8 +257,10 @@ public final class CommonFilesPanel extends javax.swing.JPanel { setTitleForSingleSource(dataSourceId); } + + CommonFilesMetaData metaData = new CommonFilesMetaData(builder.findCommonFiles(), CommonFilesPanel.this.dataSourceMap); - return builder.collateFiles(); + return metaData; } @Override @@ -266,7 +268,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel { try { super.done(); - List metadata = get(); + CommonFilesMetaData metadata = get(); CommonFilesNode commonFilesNode = new CommonFilesNode(metadata); @@ -275,15 +277,9 @@ public final class CommonFilesPanel extends javax.swing.JPanel { TableFilterNode tableFilterWithDescendantsNode = new TableFilterNode(dataResultFilterNode); - //TODO get this information from CommonFilesMetaData rather than enumerating the children as below - int totalNodes = 0; - for (CommonFilesMetaData meta : metadata) { - totalNodes += meta.getChildren().size(); - } - DataResultTopComponent component = DataResultTopComponent.createInstance(this.tabTitle); - DataResultTopComponent.initInstance(pathText, tableFilterWithDescendantsNode, 0/*totalNodes*/, component); + DataResultTopComponent.initInstance(pathText, tableFilterWithDescendantsNode, metadata.size(), component); } catch (InterruptedException ex) { LOGGER.log(Level.SEVERE, "Interrupted while loading Common Files", ex); diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetaData.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetaData.java new file mode 100644 index 0000000000..95f4147868 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetaData.java @@ -0,0 +1,48 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilesearch; + +/** + * + */ +public class FileInstanceMetaData { + + private Long objectId; + private String dataSourceName; + private Long dataSourceId; + + public FileInstanceMetaData (Long objectId, String dataSourceName, Long dataSourceId){ + this.objectId = objectId; + this.dataSourceName = dataSourceName; + this.dataSourceId = dataSourceId; + } + + public Long getObjectId(){ + return this.objectId; + } + + public String getDataSourceName(){ + return this.dataSourceName; + } + + public Long getDataSourceId(){ + return this.dataSourceId; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5MetaData.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5MetaData.java new file mode 100644 index 0000000000..c143797e70 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5MetaData.java @@ -0,0 +1,59 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilesearch; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * TODO + */ +public class Md5MetaData { + + private String md5; + private List fileInstances; + + public Md5MetaData(String md5, List fileInstances){ + this.md5 = md5; + this.fileInstances = fileInstances; + } + + public String getMd5(){ + return this.md5; + } + + public List getMetaData(){ + return this.fileInstances; + } + + public int size(){ + return this.fileInstances.size(); + } + + public String getDataSources() { + Set sources = new HashSet (); + for(FileInstanceMetaData data : this.fileInstances){ + sources.add(data.getDataSourceName()); + } + return String.join(", ", sources); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java b/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java index 390c44bed3..13240cc4cc 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java @@ -19,17 +19,25 @@ */ package org.sleuthkit.autopsy.datamodel; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; import org.openide.nodes.Node; import org.openide.nodes.Sheet; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.lookup.Lookups; -import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetaData; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.commonfilesearch.FileInstanceMetaData; +import org.sleuthkit.autopsy.commonfilesearch.Md5MetaData; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; /** * Represents a common files match - two or more files which appear to be the @@ -42,16 +50,16 @@ public class Md5Node extends DisplayableItemNode { private final int commonFileCount; private final String dataSources; - public Md5Node(CommonFilesMetaData metaData) { + public Md5Node(Md5MetaData data) { super(Children.create( - new FileInstanceNodeFactory(metaData.getChildren(), - metaData.getDataSourceIdToNameMap()), true), - Lookups.singleton(metaData.getMd5())); - this.commonFileCount = metaData.getChildren().size(); - this.dataSources = metaData.getDataSources(); - this.md5Hash = metaData.getMd5(); + new FileInstanceNodeFactory(data), true), + Lookups.singleton(data.getMd5())); + + this.commonFileCount = data.size(); + this.dataSources = String.join(", ", data.getDataSources()); + this.md5Hash = data.getMd5(); - this.setDisplayName(md5Hash); + this.setDisplayName(this.md5Hash); } int getCommonFileCount() { @@ -118,40 +126,48 @@ public class Md5Node extends DisplayableItemNode { /** * Child generator for FileInstanceNode of Md5Node. */ - static class FileInstanceNodeFactory extends ChildFactory { + static class FileInstanceNodeFactory extends ChildFactory { - private final List descendants; - private final Map dataSourceMap; + private final Md5MetaData descendants; - FileInstanceNodeFactory(List descendants, Map dataSourceMap) { + FileInstanceNodeFactory(Md5MetaData descendants) { this.descendants = descendants; - this.dataSourceMap = dataSourceMap; } @Override - protected Node createNodeForKey(AbstractFile file) { - - //TODO minimize work here - this is the UI thread - final String dataSource = this.dataSourceMap.get(file.getDataSourceObjectId()); - - return new FileInstanceNode(file, dataSource); + protected Node createNodeForKey(FileInstanceMetaData file) { + try { + Case currentCase = Case.getOpenCase(); + SleuthkitCase tskDb = currentCase.getSleuthkitCase(); + AbstractFile abstractFile = tskDb.findAllFilesWhere(String.format("obj_id in (%s)", file.getObjectId())).get(0); + + return new FileInstanceNode(abstractFile, file.getDataSourceName()); + } catch (NoCurrentCaseException ex) { + Exceptions.printStackTrace(ex); + //TODO log this + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); + //TODO log this + } + //TODO smells bad... + return null; } @Override - protected boolean createKeys(List list) { + protected boolean createKeys(List list) { - //TODO change param to Long rather than AbstractFile //TODO load children from db here + //TODO consider doing db work here??? - list.addAll(this.descendants); + list.addAll(this.descendants.getMetaData()); return true; } - @Override - protected Node createWaitNode() { - //TODO could skip this...maybe??? - return new CommonFileChildNodeLoading(Children.LEAF); - } +// @Override +// protected Node createWaitNode() { +// //TODO could skip this...maybe??? +// return new CommonFileChildNodeLoading(Children.LEAF); +// } } @NbBundle.Messages({