Merge pull request #4202 from wschaeferB/4270-FileTypeForInterCase

4270 file type for inter case

Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java

@@ -20,13 +20,16 @@
 package org.sleuthkit.autopsy.commonfilesearch;
 
 import java.sql.SQLException;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type;
+import static org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher.MEDIA_PICS_VIDEO_MIME_TYPES;
 
 /**
  * Algorithm which finds files anywhere in the Central Repo which also occur in
@@ -41,7 +44,8 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
      * @param filterByDocMimeType   match only on files whose mime types can be
      *                              broadly categorized as document types
      * @param corAttrType           attribute type
-     * @param percentageThreshold   omit any matches with frequency above this threshold
+     * @param percentageThreshold   omit any matches with frequency above this
+     *                              threshold
      *
      * @throws EamDbException
      */
@@ -53,7 +57,15 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
     public CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
         InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
         Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseCommonAttributeValues(Case.getCurrentCase());
-        return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
+        
+        Set<String> mimeTypesToFilterOn = new HashSet<>();
+        if (isFilterByMedia()) {
+            mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
+        }
+        if (isFilterByDoc()) {
+            mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
+        }
+        return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
     }
 
     @NbBundle.Messages({
@@ -62,6 +74,10 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
         "AllInterCaseCommonAttributeSearcher.buildTabTitle.titleInterAll=Common Properties (All Central Repository Cases, {0}{1})"})
     @Override
     String getTabTitle() {
-        return Bundle.AllInterCaseCommonAttributeSearcher_buildTabTitle_titleInterAll(this.corAttrType.getDisplayName(), this.getPercentThresholdString());
+        String typeString = this.corAttrType.getDisplayName();
+        if (typeString.equals("Files")) {
+            typeString = this.buildCategorySelectionString();
+        }
+        return Bundle.AllInterCaseCommonAttributeSearcher_buildTabTitle_titleInterAll(typeString, this.getPercentThresholdString());
     }
 }

Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepoCommonAttributeInstance.java

@@ -21,9 +21,7 @@ package org.sleuthkit.autopsy.commonfilesearch;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Level;
 import org.sleuthkit.autopsy.casemodule.Case;
@@ -95,7 +93,7 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
                     // Create the parent path. Make sure not to add a separator if there is already one there.
                     String parentPath = fileFromPath.getParent();
                     if (!parentPath.endsWith(File.separator)) {
-                        parentPath = parentPath + File.separator;
+                        parentPath += File.separator;
                     }
                     parentPath = parentPath.replace("\\", "/");
 
@@ -124,22 +122,15 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
 
     @Override
     public DisplayableItemNode[] generateNodes() {
-
         // @@@ We should be doing more of this work in teh generateKeys method. We want to do as little as possible in generateNodes
-        InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(correlationType);
-        CorrelationAttributeInstance corrAttr = eamDbAttrInst.findSingleCorrelationAttribute(crFileId);
         List<DisplayableItemNode> attrInstNodeList = new ArrayList<>(0);
         String currCaseDbName = Case.getCurrentCase().getDisplayName();
-
         try {
-            this.setCurrentAttributeInst(corrAttr);
-
             AbstractFile abstractFileForAttributeInstance = this.getAbstractFile();
-            DisplayableItemNode generatedInstNode = AbstractCommonAttributeInstance.createNode(corrAttr, abstractFileForAttributeInstance, currCaseDbName);
+            DisplayableItemNode generatedInstNode = AbstractCommonAttributeInstance.createNode(currentAttribute, abstractFileForAttributeInstance, currCaseDbName);
             attrInstNodeList.add(generatedInstNode);
-
         } catch (TskCoreException ex) {
-            LOGGER.log(Level.SEVERE, String.format("Unable to get DataSource for record with md5: %s.  Node not created.", new Object[]{corrAttr.getCorrelationValue()}), ex);
+            LOGGER.log(Level.SEVERE, String.format("Unable to get DataSource for record with md5: %s.  Node not created.", new Object[]{currentAttribute.getCorrelationValue()}), ex);
         }
 
         return attrInstNodeList.toArray(new DisplayableItemNode[attrInstNodeList.size()]);

Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java

@@ -258,7 +258,6 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
                 }
                 metadata = builder.findMatches();
                 this.tabTitle = builder.getTabTitle();
-
                 return metadata;
             }
 

Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributeSearchResults.java

@@ -22,15 +22,18 @@ package org.sleuthkit.autopsy.commonfilesearch;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.logging.Level;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
 import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
 import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
 import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.datamodel.AbstractFile;
 
 /**
  * Stores the results from the various types of common attribute searching
@@ -42,34 +45,44 @@ final public class CommonAttributeSearchResults {
 
     // maps instance count to list of attribute values. 
     private final Map<Integer, CommonAttributeValueList> instanceCountToAttributeValues;
-
+    private final Set<String> mimeTypesToInclude;
     private final int percentageThreshold;
     private final int resultTypeId;
 
     /**
      * Create a values object which can be handed off to the node factories.
      *
-     * @param values list of CommonAttributeValue indexed by size of
+     * @param metadata            list of CommonAttributeValue indexed by size
-     * CommonAttributeValue
+     *                            of CommonAttributeValue
+     * @param percentageThreshold threshold to filter out files which are too
+     *                            common, value of 0 is disabled
+     * @param resultType          The type of Correlation Attribute being
+     *                            searched for
+     * @param mimeTypesToFilterOn Set of mime types to include for intercase
+     *                            searches
      */
-    CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) {
+    CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
         //wrap in a new object in case any client code has used an unmodifiable collection
         this.instanceCountToAttributeValues = new HashMap<>(metadata);
         this.percentageThreshold = percentageThreshold;
         this.resultTypeId = resultType.getId();
+        this.mimeTypesToInclude = mimeTypesToFilterOn;
     }
 
     /**
      * Create a values object which can be handed off to the node factories.
      *
-     * @param values list of CommonAttributeValue indexed by size of
+     * @param metadata            list of CommonAttributeValue indexed by size
-     * CommonAttributeValue
+     *                            of CommonAttributeValue
+     * @param percentageThreshold threshold to filter out files which are too
+     *                            common, value of 0 is disabled
      */
     CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold) {
         //wrap in a new object in case any client code has used an unmodifiable collection
         this.instanceCountToAttributeValues = new HashMap<>(metadata);
         this.percentageThreshold = percentageThreshold;
         this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID;
+        this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes
     }
 
     /**
@@ -79,6 +92,7 @@ final public class CommonAttributeSearchResults {
      * <code>getValues()</code>.
      *
      * @param instanceCount key
+     *
      * @return list of values which represent matches
      */
     CommonAttributeValueList getAttributeValuesForInstanceCount(Integer instanceCount) {
@@ -93,7 +107,7 @@ final public class CommonAttributeSearchResults {
      * @return map of sizes of children to list of matches
      */
     public Map<Integer, CommonAttributeValueList> getMetadata() throws EamDbException {
-        if(this.percentageThreshold == 0){
+        if (this.percentageThreshold == 0 && mimeTypesToInclude.isEmpty()) {
             return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
         } else {
             return this.getMetadata(this.percentageThreshold);
@@ -111,11 +125,6 @@ final public class CommonAttributeSearchResults {
      * @return metadata
      */
     private Map<Integer, CommonAttributeValueList> getMetadata(int maximumPercentageThreshold) throws EamDbException {
-        
-        if(maximumPercentageThreshold == 0){
-            return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
-        }
-        
         CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
                 .getDefaultCorrelationTypes()
                 .stream()
@@ -136,12 +145,39 @@ final public class CommonAttributeSearchResults {
 
             for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata
 
+                //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list 
+                //because when performing intra case search this filtering will have been done during the query of the case database 
+                boolean mimeTypeToRemove = false;  //allow code to be more efficient by not attempting to remove the same value multiple times
+                if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
+                    for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
+                        AbstractFile abstractFile = commonAttr.getAbstractFile();
+                        if (abstractFile != null) {
+                            String mimeType = commonAttr.getAbstractFile().getMIMEType();
+                            if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
+                                if (itemsToRemove.containsKey(key)) {
+                                    itemsToRemove.get(key).add(value);
+                                } else {
+                                    List<CommonAttributeValue> toRemove = new ArrayList<>();
+                                    toRemove.add(value);
+                                    itemsToRemove.put(key, toRemove);
+                                }
+                                //value will be removed as the mime type existed and was not in the set to be included
+                                //because value is removed this value does not need to be checked further
+                                mimeTypeToRemove = true;
+                                break;  
+                            }
+                        }
+                        if (mimeTypeToRemove) {
+                            break;
+                        }
+                    }
+                }
+                if (!mimeTypeToRemove && maximumPercentageThreshold != 0) {  //only do the frequency filtering when a max % was set
                     try {
                         Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue(
                                 attributeType, value.getValue()).doubleValue();
                         Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100;
                         int frequencyPercentage = commonalityPercentage.intValue();
-                
                         if (frequencyPercentage > maximumPercentageThreshold) {
                             if (itemsToRemove.containsKey(key)) {
                                 itemsToRemove.get(key).add(value);
@@ -156,21 +192,21 @@ final public class CommonAttributeSearchResults {
                     }
                 }
             }
+        }
 
         for (Entry<Integer, List<CommonAttributeValue>> valuesToRemove : itemsToRemove.entrySet()) {
-            
             final Integer key = valuesToRemove.getKey();
             final List<CommonAttributeValue> values = valuesToRemove.getValue();
-            
             for (CommonAttributeValue value : values) {
                 final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key);
+                if (instanceCountValue != null) {
                     instanceCountValue.removeMetaData(value);
-                
                     if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata
                         this.instanceCountToAttributeValues.remove(key);
                     }
                 }
             }
+        }
 
         return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
     }

Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseCommonAttributeSearcher.java

@@ -19,7 +19,6 @@
  */
 package org.sleuthkit.autopsy.commonfilesearch;
 
-import java.util.Map;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
 import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
 import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;

Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCasePanel.java

@@ -291,8 +291,7 @@ public final class InterCasePanel extends javax.swing.JPanel {
 
 
     private void correlationTypeComboBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_correlationTypeComboBoxActionPerformed
-        //we do not currenlty have mime type in the central repository so this section will always be disabled
+        boolean enableFileTypesFilter = this.correlationTypeComboBox.getSelectedItem().equals("Files"); 
-        boolean enableFileTypesFilter = false; //this.correlationTypeComboBox.getSelectedItem().equals("Files"); 
         categoriesLabel.setEnabled(enableFileTypesFilter);
         allFileCategoriesRadioButton.setEnabled(enableFileTypesFilter);
         selectedFileCategoriesButton.setEnabled(enableFileTypesFilter);

Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseSearchResultsProcessor.java

@@ -242,7 +242,9 @@ final class InterCaseSearchResultsProcessor {
             // we don't *have* all the information for the rows in the CR,
             //  so we need to consult the present case via the SleuthkitCase object
             // Later, when the FileInstanceNode is built. Therefore, build node generators for now.
-            AbstractCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType);
+            CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType);
+            CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
+            searchResult.setCurrentAttributeInst(corrAttr);
             commonAttributeValue.addInstance(searchResult);
         }
 

Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java

@@ -20,7 +20,9 @@
 package org.sleuthkit.autopsy.commonfilesearch;
 
 import java.sql.SQLException;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
@@ -28,6 +30,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
 import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type;
+import static org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher.MEDIA_PICS_VIDEO_MIME_TYPES;
 
 /**
  *
@@ -78,8 +81,14 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
     CommonAttributeSearchResults findFiles(CorrelationCase correlationCase) throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
         InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
         Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseCommonAttributeValues(Case.getCurrentCase(), correlationCase);
-
+        Set<String> mimeTypesToFilterOn = new HashSet<>();
-        return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
+        if (isFilterByMedia()) {
+            mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
+        }
+        if (isFilterByDoc()) {
+            mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
+        }
+        return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
     }
 
     @NbBundle.Messages({
@@ -90,6 +99,10 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
 
     @Override
     String getTabTitle() {
-        return Bundle.SingleInterCaseCommonAttributeSearcher_buildTabTitle_titleInterSingle(this.correlationCaseName, this.corAttrType.getDisplayName(), this.getPercentThresholdString());
+        String typeString = this.corAttrType.getDisplayName();
+        if (typeString.equals("Files")) {
+            typeString = this.buildCategorySelectionString();
+        }
+        return Bundle.SingleInterCaseCommonAttributeSearcher_buildTabTitle_titleInterSingle(this.correlationCaseName, typeString, this.getPercentThresholdString());
     }
 }