reimplemented algorithm so that parents and children are selected separately

Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaData.java

@@ -19,44 +19,42 @@
  */
 package org.sleuthkit.autopsy.commonfilesearch;
 
-import java.sql.SQLException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
-import org.sleuthkit.datamodel.TskCoreException;
 
 /**
  * Utility and wrapper model around data required for Common Files Search results.
  * Subclass this to implement different selections of files from the case.
  */
 public class CommonFilesMetaData {
-        
+    
-    private final String parentMd5;
+    private final Map<String, Md5MetaData> metadata;
-    private final List<Long> children;
-    private final String dataSources;
     private final Map<Long, String> dataSourceIdToNameMap;
 
-    CommonFilesMetaData(String md5, List<Long> childNodes, String dataSourcesString, Map<Long,String> dataSourcesMap) throws TskCoreException, SQLException, NoCurrentCaseException {
+    CommonFilesMetaData(Map<String, Md5MetaData> metadata, Map<Long,String> dataSourcesMap) {
-        parentMd5 = md5;
+        this.metadata = metadata;
-        children = childNodes;
+        this.dataSourceIdToNameMap = dataSourcesMap;
-        dataSources = dataSourcesString;
-        dataSourceIdToNameMap = dataSourcesMap;
-    }
-
-    public String getMd5() {
-        return parentMd5;
     }
     
-    public List<Long> getChildren() {
+    public Md5MetaData getMetaDataForMd5(String md5){
-        return Collections.unmodifiableList(this.children);
+        return this.metadata.get(md5);
     }
-
+    
+    public Map<String, Md5MetaData> getMataData(){
+        return Collections.unmodifiableMap(this.metadata);
+    }
+    
     public Map<Long, String> getDataSourceIdToNameMap() {
-        return Collections.unmodifiableMap(dataSourceIdToNameMap);
+        return Collections.unmodifiableMap(this.dataSourceIdToNameMap);
     }
 
-    public String getDataSources() {
+    int size() {
-        return dataSources;
+        int count = 0;
+        for(Md5MetaData data : this.metadata.values()){
+            count += data.size();
+        }        
+        return count;
     }
 }

Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java

@@ -51,39 +51,6 @@ abstract class CommonFilesMetaDataBuilder {
         dataSourceIdToNameMap = dataSourceIdMap;
     }
 
-    private void addDataSource(Set<String> dataSources, AbstractFile file, Map<Long, String> dataSourceIdToNameMap) {
-        long datasourceId = file.getDataSourceObjectId();
-        String dataSourceName = dataSourceIdToNameMap.get(datasourceId);
-        dataSources.add(dataSourceName);
-    }
-
-    /**
-     * Sorts files in selection into a parent/child hierarchy where actual files
-     * are nested beneath a parent node which represents the common match.
-     *
-     * @return returns a reference to itself for ease of use.
-     * @throws TskCoreException
-     */
-    List<CommonFilesMetaData> collateFiles() throws TskCoreException, SQLException {
-        List<CommonFilesMetaData> metaDataModels = new ArrayList<>();
-        Map<String, Set<String>> md5ToDataSourcesStringMap = new HashMap<>();
-
-        try {
-            List<AbstractFile> files = findCommonFiles();
-
-            Map<String, List<AbstractFile>> parentNodes = new HashMap<>();
-
-            collateParentChildRelationships(files, parentNodes, md5ToDataSourcesStringMap);
-            for (String key : parentNodes.keySet()) {
-                metaDataModels.add(new CommonFilesMetaData(key, parentNodes.get(key), String.join(", ", md5ToDataSourcesStringMap.get(key)), dataSourceIdToNameMap));
-            }
-        } catch (NoCurrentCaseException ex) {
-            Exceptions.printStackTrace(ex);
-        }
-
-        return metaDataModels;
-    }
-
     protected static String SELECT_PREFIX = "SELECT obj_id, md5, data_source_obj_id from tsk_files where";
     
     /**
@@ -97,33 +64,9 @@ abstract class CommonFilesMetaDataBuilder {
      */
     protected abstract String buildSqlSelectStatement();
 
-    private void collateParentChildRelationships(List<AbstractFile> files, Map<String, List<AbstractFile>> parentNodes, Map<String, Set<String>> md5ToDataSourcesStringMap) {
+    public Map<String, Md5MetaData> findCommonFiles() throws TskCoreException, NoCurrentCaseException, SQLException {
-        for (AbstractFile file : files) {
-
-            String currentMd5 = file.getMd5Hash();
-            if((currentMd5 == null) || (HashUtility.isNoDataMd5(currentMd5))) {
-               continue; 
-            }
-            if (parentNodes.containsKey(currentMd5)) {
-                parentNodes.get(currentMd5).add(file);
-                Set<String> currentDataSources = md5ToDataSourcesStringMap.get(currentMd5);
-                addDataSource(currentDataSources, file, dataSourceIdToNameMap);
-                md5ToDataSourcesStringMap.put(currentMd5, currentDataSources);
-
-            } else {
-                List<AbstractFile> children = new ArrayList<>();
-                Set<String> dataSources = new HashSet<>();
-                children.add(file);
-                parentNodes.put(currentMd5, children);
-                addDataSource(dataSources, file, dataSourceIdToNameMap);
-                md5ToDataSourcesStringMap.put(currentMd5, dataSources);
-            }
-        }
-    }
-
-    private Map<String, List<Long>> findCommonFiles() throws TskCoreException, NoCurrentCaseException, SQLException {
         
-        Map<String, List<Long>> md5ToObjIdMap = new HashMap<>();
+        Map<String, Md5MetaData> commonFiles = new HashMap<>();
         
         SleuthkitCase sleuthkitCase = Case.getOpenCase().getSleuthkitCase();
         String selectStatement = this.buildSqlSelectStatement();
@@ -131,19 +74,22 @@ abstract class CommonFilesMetaDataBuilder {
         try (CaseDbQuery query = sleuthkitCase.executeQuery(selectStatement)){
             ResultSet resultSet = query.getResultSet();
             while(resultSet.next()){
-                String md5 = resultSet.getString(1);
+                Long objectId = resultSet.getLong(1);
-                Long objectId = resultSet.getLong(2);
+                String md5 = resultSet.getString(2);
                 Long dataSourceId = resultSet.getLong(3);
+                String dataSource = this.dataSourceIdToNameMap.get(dataSourceId);
                 
-                if(md5ToObjIdMap.containsKey(md5)){
+                if(commonFiles.containsKey(md5)){
-                    md5ToObjIdMap.get(md5).add(objectId);
+                    commonFiles.get(md5).getMetaData().add(new FileInstanceMetaData(objectId, dataSource, dataSourceId));
                 } else {
-                    List<Long> objectIds = new ArrayList<>();
+                    List<FileInstanceMetaData> fileInstances = new ArrayList<>();
-                    md5ToObjIdMap.put(md5, objectIds);
+                    fileInstances.add(new FileInstanceMetaData(objectId, dataSource, dataSourceId));
+                    Md5MetaData md5s = new Md5MetaData(md5, fileInstances);
+                    commonFiles.put(md5, md5s);
                 }
             }
         }        
         
-        return md5ToObjIdMap;
+        return commonFiles;
     }
 }

Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesNode.java

@@ -34,7 +34,7 @@ import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor;
  */
 final public class CommonFilesNode extends DisplayableItemNode {
     
-    CommonFilesNode(List<CommonFilesMetaData> metaDataList) {
+    CommonFilesNode(CommonFilesMetaData metaDataList) {
         super(Children.create(new Md5NodeFactory(metaDataList), true), Lookups.singleton(CommonFilesNode.class));
     }
 
@@ -64,31 +64,31 @@ final public class CommonFilesNode extends DisplayableItemNode {
      * ChildFactory which builds CommonFileParentNodes from the
      * CommonFilesMetaaData models.
      */
-    static class Md5NodeFactory extends ChildFactory<CommonFilesMetaData> {
+    static class Md5NodeFactory extends ChildFactory<String> {
 
         /**
          * List of models, each of which is a parent node matching a single md5,
          * containing children FileNodes.
          */
-        private List<CommonFilesMetaData> metaDataList;
+        private CommonFilesMetaData metadata;
 
-        Md5NodeFactory(List<CommonFilesMetaData> theMetaDataList) {
+        Md5NodeFactory(CommonFilesMetaData theMetaDataList) {
-            this.metaDataList = theMetaDataList;
+            this.metadata = theMetaDataList;
         }
 
         protected void removeNotify() {
-            metaDataList = null;
+            metadata = null;
         }
-
+        
         @Override
-        protected Node createNodeForKey(CommonFilesMetaData metaData) {
+        protected Node createNodeForKey(String md5){
-
+            Md5MetaData metaData = this.metadata.getMetaDataForMd5(md5);
             return new Md5Node(metaData);
         }
 
         @Override
-        protected boolean createKeys(List<CommonFilesMetaData> toPopulate) {
+        protected boolean createKeys(List<String> list) {
-            toPopulate.addAll(metaDataList);
+            list.addAll(this.metadata.getMataData().keySet());
             return true;
         }
     }

Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java

@@ -213,7 +213,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
 
         String pathText = Bundle.CommonFilesPanel_search_results_pathText();
 
-        new SwingWorker<List<CommonFilesMetaData>, Void>() {
+        new SwingWorker<CommonFilesMetaData, Void>() {
 
             private String tabTitle;
 
@@ -243,7 +243,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
 
             @Override
             @SuppressWarnings({"BoxedValueEquality", "NumberEquality"})
-            protected List<CommonFilesMetaData> doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException {
+            protected CommonFilesMetaData doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException {
                 Long dataSourceId = determineDataSourceId();
 
                 CommonFilesMetaDataBuilder builder;
@@ -257,8 +257,10 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
 
                     setTitleForSingleSource(dataSourceId);
                 }
+                
+                CommonFilesMetaData metaData = new CommonFilesMetaData(builder.findCommonFiles(), CommonFilesPanel.this.dataSourceMap);
 
-                return builder.collateFiles();
+                return metaData;
             }
 
             @Override
@@ -266,7 +268,7 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
                 try {
                     super.done();
 
-                    List<CommonFilesMetaData> metadata = get();
+                    CommonFilesMetaData metadata = get();
 
                     CommonFilesNode commonFilesNode = new CommonFilesNode(metadata);
 
@@ -275,15 +277,9 @@ public final class CommonFilesPanel extends javax.swing.JPanel {
 
                     TableFilterNode tableFilterWithDescendantsNode = new TableFilterNode(dataResultFilterNode);
 
-                    //TODO get this information from CommonFilesMetaData rather than enumerating the children as below
-                    int totalNodes = 0;
-                    for (CommonFilesMetaData meta : metadata) {
-                        totalNodes += meta.getChildren().size();
-                    }
-
                     DataResultTopComponent component = DataResultTopComponent.createInstance(this.tabTitle);
 
-                    DataResultTopComponent.initInstance(pathText, tableFilterWithDescendantsNode, 0/*totalNodes*/, component);
+                    DataResultTopComponent.initInstance(pathText, tableFilterWithDescendantsNode, metadata.size(), component);
 
                 } catch (InterruptedException ex) {
                     LOGGER.log(Level.SEVERE, "Interrupted while loading Common Files", ex);

Core/src/org/sleuthkit/autopsy/commonfilesearch/FileInstanceMetaData.java

@@ -0,0 +1,48 @@
+/*
+ * 
+ * Autopsy Forensic Browser
+ * 
+ * Copyright 2018 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.commonfilesearch;
+
+/**
+ * 
+ */
+public class FileInstanceMetaData {
+    
+    private Long objectId;
+    private String dataSourceName;
+    private Long dataSourceId;
+    
+    public FileInstanceMetaData (Long objectId, String dataSourceName, Long dataSourceId){
+        this.objectId = objectId;
+        this.dataSourceName = dataSourceName;
+        this.dataSourceId = dataSourceId;
+    }
+    
+    public Long getObjectId(){
+        return this.objectId;
+    }
+    
+    public String getDataSourceName(){
+        return this.dataSourceName;
+    }
+    
+    public Long getDataSourceId(){
+        return this.dataSourceId;
+    }    
+}

Core/src/org/sleuthkit/autopsy/commonfilesearch/Md5MetaData.java

@@ -0,0 +1,59 @@
+/*
+ * 
+ * Autopsy Forensic Browser
+ * 
+ * Copyright 2018 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.commonfilesearch;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * TODO
+ */
+public class Md5MetaData {
+    
+    private String md5;
+    private List<FileInstanceMetaData> fileInstances;
+    
+    public Md5MetaData(String md5, List<FileInstanceMetaData> fileInstances){
+        this.md5 = md5;
+        this.fileInstances = fileInstances;
+    }
+    
+    public String getMd5(){
+        return this.md5;
+    }
+    
+    public List<FileInstanceMetaData> getMetaData(){
+        return this.fileInstances;
+    }
+    
+    public int size(){
+        return this.fileInstances.size();
+    }
+
+    public String getDataSources() {
+        Set<String> sources = new HashSet<String> ();
+        for(FileInstanceMetaData data  : this.fileInstances){
+            sources.add(data.getDataSourceName());
+        }
+        return String.join(", ", sources);
+    }
+}

Core/src/org/sleuthkit/autopsy/datamodel/Md5Node.java

@@ -19,17 +19,25 @@
  */
 package org.sleuthkit.autopsy.datamodel;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.openide.nodes.ChildFactory;
 import org.openide.nodes.Children;
 import org.openide.nodes.Node;
 import org.openide.nodes.Sheet;
+import org.openide.util.Exceptions;
 import org.openide.util.NbBundle;
 import org.openide.util.lookup.Lookups;
-import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetaData;
+import org.sleuthkit.autopsy.casemodule.Case;
+import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
+import org.sleuthkit.autopsy.commonfilesearch.FileInstanceMetaData;
+import org.sleuthkit.autopsy.commonfilesearch.Md5MetaData;
 import org.sleuthkit.datamodel.AbstractFile;
+import org.sleuthkit.datamodel.SleuthkitCase;
+import org.sleuthkit.datamodel.TskCoreException;
 
 /**
  * Represents a common files match - two or more files which appear to be the
@@ -42,16 +50,16 @@ public class Md5Node extends DisplayableItemNode {
     private final int commonFileCount;
     private final String dataSources;
 
-    public Md5Node(CommonFilesMetaData metaData) {
+    public Md5Node(Md5MetaData data) {
         super(Children.create(
-                new FileInstanceNodeFactory(metaData.getChildren(),
+                new FileInstanceNodeFactory(data), true),
-                        metaData.getDataSourceIdToNameMap()), true),
+                Lookups.singleton(data.getMd5()));
-                Lookups.singleton(metaData.getMd5()));
+        
-        this.commonFileCount = metaData.getChildren().size();
+        this.commonFileCount = data.size();
-        this.dataSources = metaData.getDataSources();
+        this.dataSources = String.join(", ", data.getDataSources());
-        this.md5Hash = metaData.getMd5();
+        this.md5Hash = data.getMd5();
 
-        this.setDisplayName(md5Hash);
+        this.setDisplayName(this.md5Hash);
     }
 
     int getCommonFileCount() {
@@ -118,40 +126,48 @@ public class Md5Node extends DisplayableItemNode {
     /**
      * Child generator for <code>FileInstanceNode</code> of <code>Md5Node</code>.
      */
-    static class FileInstanceNodeFactory extends ChildFactory<AbstractFile> {
+    static class FileInstanceNodeFactory extends ChildFactory<FileInstanceMetaData> {
 
-        private final List<AbstractFile> descendants;
+        private final Md5MetaData descendants;
-        private final Map<Long, String> dataSourceMap;
 
-        FileInstanceNodeFactory(List<AbstractFile> descendants, Map<Long, String> dataSourceMap) {
+        FileInstanceNodeFactory(Md5MetaData descendants) {
             this.descendants = descendants;
-            this.dataSourceMap = dataSourceMap;
         }
 
         @Override
-        protected Node createNodeForKey(AbstractFile file) {
+        protected Node createNodeForKey(FileInstanceMetaData file) {
-
+            try {
-            //TODO minimize work here - this is the UI thread
+                Case currentCase = Case.getOpenCase();
-            final String dataSource = this.dataSourceMap.get(file.getDataSourceObjectId());
+                SleuthkitCase tskDb = currentCase.getSleuthkitCase();
-
+                AbstractFile abstractFile = tskDb.findAllFilesWhere(String.format("obj_id in (%s)", file.getObjectId())).get(0);
-            return new FileInstanceNode(file, dataSource);
+                
+                return new FileInstanceNode(abstractFile, file.getDataSourceName());
+            } catch (NoCurrentCaseException ex) {
+                Exceptions.printStackTrace(ex);
+                //TODO log this
+            } catch (TskCoreException ex) {
+                Exceptions.printStackTrace(ex);
+                //TODO log this
+            }
+            //TODO smells bad...
+            return null;
         }
 
         @Override
-        protected boolean createKeys(List<AbstractFile> list) {
+        protected boolean createKeys(List<FileInstanceMetaData> list) {
             
-            //TODO change param to Long rather than AbstractFile
             //TODO load children from db here
+            //TODO consider doing db work here???
             
-            list.addAll(this.descendants);
+            list.addAll(this.descendants.getMetaData());
             return true;
         }
 
-        @Override
+//        @Override
-        protected Node createWaitNode() {
+//        protected Node createWaitNode() {
-            //TODO could skip this...maybe???
+//            //TODO could skip this...maybe???
-            return new CommonFileChildNodeLoading(Children.LEAF);
+//            return new CommonFileChildNodeLoading(Children.LEAF);
-        }
+//        }
     }
 
     @NbBundle.Messages({