Merge pull request #4202 from wschaeferB/4270-FileTypeForInterCase

4270 file type for inter case
This commit is contained in:
Brian Carrier 2018-10-25 11:35:04 -04:00 committed by GitHub
commit 0a5a6b07b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 150 additions and 95 deletions

View File

@ -20,13 +20,16 @@
package org.sleuthkit.autopsy.commonfilesearch;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type;
import static org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher.MEDIA_PICS_VIDEO_MIME_TYPES;
/**
* Algorithm which finds files anywhere in the Central Repo which also occur in
@ -41,7 +44,8 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
* @param filterByDocMimeType match only on files whose mime types can be
* broadly categorized as document types
* @param corAttrType attribute type
* @param percentageThreshold omit any matches with frequency above this threshold
* @param percentageThreshold omit any matches with frequency above this
* threshold
*
* @throws EamDbException
*/
@ -53,7 +57,15 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
public CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseCommonAttributeValues(Case.getCurrentCase());
return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
Set<String> mimeTypesToFilterOn = new HashSet<>();
if (isFilterByMedia()) {
mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
}
if (isFilterByDoc()) {
mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
}
return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
}
@NbBundle.Messages({
@ -62,6 +74,10 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
"AllInterCaseCommonAttributeSearcher.buildTabTitle.titleInterAll=Common Properties (All Central Repository Cases, {0}{1})"})
@Override
String getTabTitle() {
return Bundle.AllInterCaseCommonAttributeSearcher_buildTabTitle_titleInterAll(this.corAttrType.getDisplayName(), this.getPercentThresholdString());
String typeString = this.corAttrType.getDisplayName();
if (typeString.equals("Files")) {
typeString = this.buildCategorySelectionString();
}
return Bundle.AllInterCaseCommonAttributeSearcher_buildTabTitle_titleInterAll(typeString, this.getPercentThresholdString());
}
}

View File

@ -21,9 +21,7 @@ package org.sleuthkit.autopsy.commonfilesearch;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
@ -55,7 +53,7 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
}
@Override
public CorrelationAttributeInstance.Type getCorrelationAttributeInstanceType(){
public CorrelationAttributeInstance.Type getCorrelationAttributeInstanceType() {
return this.correlationType;
}
@ -84,7 +82,7 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
Optional<DataSource> dataSource = tskDb.getDataSources().stream()
.filter(p -> p.getDeviceId().equals(currentAttribute.getCorrelationDataSource().getDeviceID()))
.findFirst();
if (! dataSource.isPresent()) {
if (!dataSource.isPresent()) {
LOGGER.log(Level.WARNING, String.format("Unable to find data source with device ID %s in the current case", currentAttribute.getCorrelationDataSource().getDeviceID()));
return null;
}
@ -94,8 +92,8 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
// Create the parent path. Make sure not to add a separator if there is already one there.
String parentPath = fileFromPath.getParent();
if (! parentPath.endsWith(File.separator)) {
parentPath = parentPath + File.separator;
if (!parentPath.endsWith(File.separator)) {
parentPath += File.separator;
}
parentPath = parentPath.replace("\\", "/");
@ -124,22 +122,15 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
@Override
public DisplayableItemNode[] generateNodes() {
// @@@ We should be doing more of this work in teh generateKeys method. We want to do as little as possible in generateNodes
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(correlationType);
CorrelationAttributeInstance corrAttr = eamDbAttrInst.findSingleCorrelationAttribute(crFileId);
List<DisplayableItemNode> attrInstNodeList = new ArrayList<>(0);
String currCaseDbName = Case.getCurrentCase().getDisplayName();
try {
this.setCurrentAttributeInst(corrAttr);
AbstractFile abstractFileForAttributeInstance = this.getAbstractFile();
DisplayableItemNode generatedInstNode = AbstractCommonAttributeInstance.createNode(corrAttr, abstractFileForAttributeInstance, currCaseDbName);
DisplayableItemNode generatedInstNode = AbstractCommonAttributeInstance.createNode(currentAttribute, abstractFileForAttributeInstance, currCaseDbName);
attrInstNodeList.add(generatedInstNode);
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, String.format("Unable to get DataSource for record with md5: %s. Node not created.", new Object[]{corrAttr.getCorrelationValue()}), ex);
LOGGER.log(Level.SEVERE, String.format("Unable to get DataSource for record with md5: %s. Node not created.", new Object[]{currentAttribute.getCorrelationValue()}), ex);
}
return attrInstNodeList.toArray(new DisplayableItemNode[attrInstNodeList.size()]);

View File

@ -258,7 +258,6 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
}
metadata = builder.findMatches();
this.tabTitle = builder.getTabTitle();
return metadata;
}

View File

@ -22,15 +22,18 @@ package org.sleuthkit.autopsy.commonfilesearch;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Stores the results from the various types of common attribute searching
@ -42,34 +45,44 @@ final public class CommonAttributeSearchResults {
// maps instance count to list of attribute values.
private final Map<Integer, CommonAttributeValueList> instanceCountToAttributeValues;
private final Set<String> mimeTypesToInclude;
private final int percentageThreshold;
private final int resultTypeId;
/**
* Create a values object which can be handed off to the node factories.
*
* @param values list of CommonAttributeValue indexed by size of
* CommonAttributeValue
* @param metadata list of CommonAttributeValue indexed by size
* of CommonAttributeValue
* @param percentageThreshold threshold to filter out files which are too
* common, value of 0 is disabled
* @param resultType The type of Correlation Attribute being
* searched for
* @param mimeTypesToFilterOn Set of mime types to include for intercase
* searches
*/
CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) {
CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
//wrap in a new object in case any client code has used an unmodifiable collection
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
this.resultTypeId = resultType.getId();
this.mimeTypesToInclude = mimeTypesToFilterOn;
}
/**
* Create a values object which can be handed off to the node factories.
*
* @param values list of CommonAttributeValue indexed by size of
* CommonAttributeValue
* @param metadata list of CommonAttributeValue indexed by size
* of CommonAttributeValue
* @param percentageThreshold threshold to filter out files which are too
* common, value of 0 is disabled
*/
CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold) {
//wrap in a new object in case any client code has used an unmodifiable collection
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID;
this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes
}
/**
@ -79,6 +92,7 @@ final public class CommonAttributeSearchResults {
* <code>getValues()</code>.
*
* @param instanceCount key
*
* @return list of values which represent matches
*/
CommonAttributeValueList getAttributeValuesForInstanceCount(Integer instanceCount) {
@ -93,7 +107,7 @@ final public class CommonAttributeSearchResults {
* @return map of sizes of children to list of matches
*/
public Map<Integer, CommonAttributeValueList> getMetadata() throws EamDbException {
if(this.percentageThreshold == 0){
if (this.percentageThreshold == 0 && mimeTypesToInclude.isEmpty()) {
return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
} else {
return this.getMetadata(this.percentageThreshold);
@ -111,11 +125,6 @@ final public class CommonAttributeSearchResults {
* @return metadata
*/
private Map<Integer, CommonAttributeValueList> getMetadata(int maximumPercentageThreshold) throws EamDbException {
if(maximumPercentageThreshold == 0){
return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
}
CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
.getDefaultCorrelationTypes()
.stream()
@ -129,21 +138,48 @@ final public class CommonAttributeSearchResults {
//the frequencyPercentage
Double uniqueCaseDataSourceTuples = eamDb.getCountUniqueDataSources().doubleValue();
for(Entry<Integer, CommonAttributeValueList> listOfValues : Collections.unmodifiableMap(this.instanceCountToAttributeValues).entrySet()){
for (Entry<Integer, CommonAttributeValueList> listOfValues : Collections.unmodifiableMap(this.instanceCountToAttributeValues).entrySet()) {
final Integer key = listOfValues.getKey();
final CommonAttributeValueList values = listOfValues.getValue();
for(CommonAttributeValue value : values.getDelayedMetadataList()){ // Need the real metadata
for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata
//Intracase common attribute searches will have been created with an empty mimeTypesToInclude list
//because when performing intra case search this filtering will have been done during the query of the case database
boolean mimeTypeToRemove = false; //allow code to be more efficient by not attempting to remove the same value multiple times
if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
AbstractFile abstractFile = commonAttr.getAbstractFile();
if (abstractFile != null) {
String mimeType = commonAttr.getAbstractFile().getMIMEType();
if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
if (itemsToRemove.containsKey(key)) {
itemsToRemove.get(key).add(value);
} else {
List<CommonAttributeValue> toRemove = new ArrayList<>();
toRemove.add(value);
itemsToRemove.put(key, toRemove);
}
//value will be removed as the mime type existed and was not in the set to be included
//because value is removed this value does not need to be checked further
mimeTypeToRemove = true;
break;
}
}
if (mimeTypeToRemove) {
break;
}
}
}
if (!mimeTypeToRemove && maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
try {
Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue(
attributeType, value.getValue()).doubleValue();
Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100;
int frequencyPercentage = commonalityPercentage.intValue();
if(frequencyPercentage > maximumPercentageThreshold){
if(itemsToRemove.containsKey(key)){
if (frequencyPercentage > maximumPercentageThreshold) {
if (itemsToRemove.containsKey(key)) {
itemsToRemove.get(key).add(value);
} else {
List<CommonAttributeValue> toRemove = new ArrayList<>();
@ -151,26 +187,26 @@ final public class CommonAttributeSearchResults {
itemsToRemove.put(key, toRemove);
}
}
} catch(CorrelationAttributeNormalizationException ex){
} catch (CorrelationAttributeNormalizationException ex) {
LOGGER.log(Level.WARNING, "Unable to determine frequency percentage attribute - frequency filter may not be accurate for these results.", ex);
}
}
}
}
for(Entry<Integer, List<CommonAttributeValue>> valuesToRemove : itemsToRemove.entrySet()){
for (Entry<Integer, List<CommonAttributeValue>> valuesToRemove : itemsToRemove.entrySet()) {
final Integer key = valuesToRemove.getKey();
final List<CommonAttributeValue> values = valuesToRemove.getValue();
for (CommonAttributeValue value : values){
for (CommonAttributeValue value : values) {
final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key);
if (instanceCountValue != null) {
instanceCountValue.removeMetaData(value);
if(instanceCountValue.getDelayedMetadataList().isEmpty()){ // Check the real metadata
if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata
this.instanceCountToAttributeValues.remove(key);
}
}
}
}
return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
}
@ -184,7 +220,7 @@ final public class CommonAttributeSearchResults {
int count = 0;
for (CommonAttributeValueList data : this.instanceCountToAttributeValues.values()) {
for(CommonAttributeValue md5 : data.getDelayedMetadataList()){
for (CommonAttributeValue md5 : data.getDelayedMetadataList()) {
count += md5.getInstanceCount();
}
}

View File

@ -19,7 +19,6 @@
*/
package org.sleuthkit.autopsy.commonfilesearch;
import java.util.Map;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;

View File

@ -291,8 +291,7 @@ public final class InterCasePanel extends javax.swing.JPanel {
private void correlationTypeComboBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_correlationTypeComboBoxActionPerformed
//we do not currenlty have mime type in the central repository so this section will always be disabled
boolean enableFileTypesFilter = false; //this.correlationTypeComboBox.getSelectedItem().equals("Files");
boolean enableFileTypesFilter = this.correlationTypeComboBox.getSelectedItem().equals("Files");
categoriesLabel.setEnabled(enableFileTypesFilter);
allFileCategoriesRadioButton.setEnabled(enableFileTypesFilter);
selectedFileCategoriesButton.setEnabled(enableFileTypesFilter);

View File

@ -242,7 +242,9 @@ final class InterCaseSearchResultsProcessor {
// we don't *have* all the information for the rows in the CR,
// so we need to consult the present case via the SleuthkitCase object
// Later, when the FileInstanceNode is built. Therefore, build node generators for now.
AbstractCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType);
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType);
CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
searchResult.setCurrentAttributeInst(corrAttr);
commonAttributeValue.addInstance(searchResult);
}

View File

@ -20,7 +20,9 @@
package org.sleuthkit.autopsy.commonfilesearch;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
@ -28,6 +30,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type;
import static org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher.MEDIA_PICS_VIDEO_MIME_TYPES;
/**
*
@ -78,8 +81,14 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
CommonAttributeSearchResults findFiles(CorrelationCase correlationCase) throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseCommonAttributeValues(Case.getCurrentCase(), correlationCase);
return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
Set<String> mimeTypesToFilterOn = new HashSet<>();
if (isFilterByMedia()) {
mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
}
if (isFilterByDoc()) {
mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
}
return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
}
@NbBundle.Messages({
@ -90,6 +99,10 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
@Override
String getTabTitle() {
return Bundle.SingleInterCaseCommonAttributeSearcher_buildTabTitle_titleInterSingle(this.correlationCaseName, this.corAttrType.getDisplayName(), this.getPercentThresholdString());
String typeString = this.corAttrType.getDisplayName();
if (typeString.equals("Files")) {
typeString = this.buildCategorySelectionString();
}
return Bundle.SingleInterCaseCommonAttributeSearcher_buildTabTitle_titleInterSingle(this.correlationCaseName, typeString, this.getPercentThresholdString());
}
}