Merge branch 'develop' of https://github.com/sleuthkit/autopsy into 1205-download-source

This commit is contained in:
Raman 2019-03-05 09:27:54 -05:00
commit da28fabe13
10 changed files with 418 additions and 336 deletions

View File

@ -34,6 +34,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.time.LocalDate;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@ -625,7 +626,7 @@ abstract class AbstractSqlEamDb implements EamDb {
// This data source is already in the central repo
return eamDataSource;
}
Connection conn = connect();
PreparedStatement preparedStatement = null;
@ -650,7 +651,7 @@ abstract class AbstractSqlEamDb implements EamDb {
/*
* If nothing was inserted, then return the data source that
* exists in the Central Repository.
*
*
* This is expected to occur with PostgreSQL Central Repository
* databases.
*/
@ -675,7 +676,7 @@ abstract class AbstractSqlEamDb implements EamDb {
* If an exception was thrown causing us to not return a new data
* source, attempt to get an existing data source with the same case
* ID and data source object ID.
*
*
* This exception block is expected to occur with SQLite Central
* Repository databases.
*/
@ -1052,30 +1053,43 @@ abstract class AbstractSqlEamDb implements EamDb {
}
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValue of the given eamArtifact.
*
* @param aType The type of the artifact
* @param value The correlation value
*
* @return List of artifact instances for a given type/value
*
* @throws EamDbException
*/
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException {
return getArtifactInstancesByTypeValues(aType, Arrays.asList(value));
}
String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value);
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List<String> values) throws EamDbException, CorrelationAttributeNormalizationException {
return getArtifactInstances(prepareGetInstancesSql(aType, values), aType);
}
Connection conn = connect();
List<CorrelationAttributeInstance> artifactInstances = new ArrayList<>();
CorrelationAttributeInstance artifactInstance;
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List<String> values, List<Integer> caseIds) throws EamDbException, CorrelationAttributeNormalizationException {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType);
String sql
= " and "
+ tableName
+ ".case_id in ('";
StringBuilder inValuesBuilder = new StringBuilder(prepareGetInstancesSql(aType, values));
inValuesBuilder.append(sql);
inValuesBuilder.append(caseIds.stream().map(String::valueOf).collect(Collectors.joining("', '")));
inValuesBuilder.append("')");
return getArtifactInstances(inValuesBuilder.toString(), aType);
}
/**
* Get the select statement for retrieving correlation attribute instances
* from the CR for a given type with values matching the specified values
*
* @param aType The type of the artifact
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
*
* @return the select statement as a String
*
* @throws CorrelationAttributeNormalizationException
*/
private String prepareGetInstancesSql(CorrelationAttributeInstance.Type aType, List<String> values) throws CorrelationAttributeNormalizationException {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType);
String sql
= "SELECT "
@ -1093,11 +1107,42 @@ abstract class AbstractSqlEamDb implements EamDb {
+ " LEFT JOIN data_sources ON "
+ tableName
+ ".data_source_id=data_sources.id"
+ " WHERE value=?";
+ " WHERE value IN (";
StringBuilder inValuesBuilder = new StringBuilder(sql);
for (String value : values) {
if (value != null) {
inValuesBuilder.append("'");
inValuesBuilder.append(CorrelationAttributeNormalizer.normalize(aType, value));
inValuesBuilder.append("',");
}
}
inValuesBuilder.deleteCharAt(inValuesBuilder.length() - 1); //delete last comma
inValuesBuilder.append(")");
return inValuesBuilder.toString();
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValues of the given eamArtifact.
*
* @param aType The type of the artifact
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
*
* @return List of artifact instances for a given type with the specified
* values
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
private List<CorrelationAttributeInstance> getArtifactInstances(String sql, CorrelationAttributeInstance.Type aType) throws CorrelationAttributeNormalizationException, EamDbException {
Connection conn = connect();
List<CorrelationAttributeInstance> artifactInstances = new ArrayList<>();
CorrelationAttributeInstance artifactInstance;
PreparedStatement preparedStatement = null;
ResultSet resultSet = null;
try {
preparedStatement = conn.prepareStatement(sql);
preparedStatement.setString(1, normalizedValue);
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
artifactInstance = getEamArtifactInstanceFromResultSet(resultSet, aType);
@ -1110,7 +1155,6 @@ abstract class AbstractSqlEamDb implements EamDb {
EamDbUtil.closeResultSet(resultSet);
EamDbUtil.closeConnection(conn);
}
return artifactInstances;
}

View File

@ -24,7 +24,6 @@ import java.util.Set;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
/**
* Main interface for interacting with the database
@ -200,27 +199,29 @@ public interface EamDb {
* Creates new Data Source in the database
*
* @param eamDataSource the data source to add
*
* @return - A CorrelationDataSource object with data source's central repository id
*
* @return - A CorrelationDataSource object with data source's central
* repository id
*/
CorrelationDataSource newDataSource(CorrelationDataSource eamDataSource) throws EamDbException;
/**
* Updates the MD5 hash value in an existing data source in the database.
*
* @param eamDataSource The data source to update
*/
void updateDataSourceMd5Hash(CorrelationDataSource eamDataSource) throws EamDbException;
/**
* Updates the SHA-1 hash value in an existing data source in the database.
*
* @param eamDataSource The data source to update
*/
void updateDataSourceSha1Hash(CorrelationDataSource eamDataSource) throws EamDbException;
/**
* Updates the SHA-256 hash value in an existing data source in the database.
* Updates the SHA-256 hash value in an existing data source in the
* database.
*
* @param eamDataSource The data source to update
*/
@ -257,14 +258,14 @@ public interface EamDb {
/**
* Changes the name of a data source in the DB
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*/
void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException;
/**
* Inserts new Artifact(s) into the database. Should add associated Case and
* Data Source first.
@ -273,17 +274,55 @@ public interface EamDb {
*/
void addArtifactInstance(CorrelationAttributeInstance eamArtifact) throws EamDbException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValues of the given eamArtifact.
*
* @param aType EamArtifact.Type to search for
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
*
* @return List of artifact instances for a given type with the specified
* values
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
List<CorrelationAttributeInstance> getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List<String> values) throws EamDbException, CorrelationAttributeNormalizationException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValue of the given eamArtifact.
*
* @param aType EamArtifact.Type to search for
* @param value Value to search for
* @param aType The type of the artifact
* @param value The correlation value
*
* @return List of artifact instances for a given type/value
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
List<CorrelationAttributeInstance> getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValues of the given eamArtifact
* for the specified cases.
*
* @param aType The type of the artifact
* @param values The list of correlation values to get
* CorrelationAttributeInstances for
* @param caseIds The list of central repository case ids to get
* CorrelationAttributeInstances for
*
* @return List of artifact instances for a given type with the specified
* values for the specified cases
*
* @throws CorrelationAttributeNormalizationException
* @throws EamDbException
*/
List<CorrelationAttributeInstance> getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List<String> values, List<Integer> caseIds) throws EamDbException, CorrelationAttributeNormalizationException;
/**
* Retrieves eamArtifact instances from the database that are associated
* with the aType and filePath
@ -341,7 +380,7 @@ public interface EamDb {
* Retrieves number of eamArtifact instances in the database that are
* associated with the given data source.
*
* @param correlationDataSource Data source to search for
* @param correlationDataSource Data source to search for
*
* @return Number of artifact instances having caseDisplayName and
* dataSource

View File

@ -275,7 +275,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
}
@Override
public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException{
public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException {
try {
acquireExclusiveLock();
super.addDataSourceObjectId(rowId, dataSourceObjectId);
@ -433,14 +433,14 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseSharedLock();
}
}
/**
* Changes the name of a data source in the DB
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*
* @param eamDataSource The data source
* @param newName The new name
*
* @throws EamDbException
*/
@Override
public void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException {
@ -451,7 +451,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseExclusiveLock();
}
}
/**
* Updates the MD5 hash value in an existing data source in the database.
*
@ -466,7 +466,7 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseExclusiveLock();
}
}
/**
* Updates the SHA-1 hash value in an existing data source in the database.
*
@ -481,9 +481,10 @@ final class SqliteEamDb extends AbstractSqlEamDb {
releaseExclusiveLock();
}
}
/**
* Updates the SHA-256 hash value in an existing data source in the database.
* Updates the SHA-256 hash value in an existing data source in the
* database.
*
* @param eamDataSource The data source to update
*/
@ -513,15 +514,6 @@ final class SqliteEamDb extends AbstractSqlEamDb {
}
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the eamArtifactType and eamArtifactValue of the given eamArtifact.
*
* @param aType The type of the artifact
* @param value The correlation value
*
* @return List of artifact instances for a given type/value
*/
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException {
try {
@ -532,6 +524,26 @@ final class SqliteEamDb extends AbstractSqlEamDb {
}
}
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List<String> values) throws EamDbException, CorrelationAttributeNormalizationException {
try {
acquireSharedLock();
return super.getArtifactInstancesByTypeValues(aType, values);
} finally {
releaseSharedLock();
}
}
@Override
public List<CorrelationAttributeInstance> getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List<String> values, List<Integer> caseIds) throws EamDbException, CorrelationAttributeNormalizationException {
try {
acquireSharedLock();
return super.getArtifactInstancesByTypeValuesAndCases(aType, values, caseIds);
} finally {
releaseSharedLock();
}
}
/**
* Retrieves eamArtifact instances from the database that are associated
* with the aType and filePath

View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -56,7 +56,6 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
@Override
public CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase());
Set<String> mimeTypesToFilterOn = new HashSet<>();
if (isFilterByMedia()) {
mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
@ -64,13 +63,13 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
if (isFilterByDoc()) {
mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
}
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn);
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
@Override
public CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase());
Set<String> mimeTypesToFilterOn = new HashSet<>();
if (isFilterByMedia()) {
mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
@ -78,7 +77,8 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
if (isFilterByDoc()) {
mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
}
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn);
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
@NbBundle.Messages({

View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -33,7 +33,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNor
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Stores the results from the various types of common attribute searching
@ -55,11 +54,9 @@ final public class CommonAttributeCaseSearchResults {
* common, value of 0 is disabled
* @param resultType The type of Correlation Attribute being
* searched for
* @param mimeTypesToFilterOn Set of mime types to include for intercase
* searches
*/
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId(), mimeTypesToFilterOn);
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId());
}
/**
@ -71,7 +68,7 @@ final public class CommonAttributeCaseSearchResults {
* common, value of 0 is disabled
*/
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID, new HashSet<>());
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID);
}
/**
@ -110,11 +107,10 @@ final public class CommonAttributeCaseSearchResults {
* not be more common than
* @param resultTypeId the ID of the result type contained in the
* metadata
* @param mimeTypesToFilterOn the mimetypes to include in our results
*
* @return metadata
*/
private Map<String, Map<String, CommonAttributeValueList>> filterMetadata(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, int resultTypeId, Set<String> mimeTypesToFilterOn) {
private Map<String, Map<String, CommonAttributeValueList>> filterMetadata(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, int resultTypeId) {
try {
final String currentCaseName;
try {
@ -123,8 +119,9 @@ final public class CommonAttributeCaseSearchResults {
throw new EamDbException("Unable to get current case while performing filtering", ex);
}
Map<String, CommonAttributeValueList> currentCaseDataSourceMap = metadata.get(currentCaseName);
if (currentCaseDataSourceMap == null) {
throw new EamDbException("No data for current case found in results, indicating there are no results and nothing will be filtered");
Map<String, Map<String, CommonAttributeValueList>> filteredCaseNameToDataSourcesTree = new HashMap<>();
if (currentCaseDataSourceMap == null) { //there are no results
return filteredCaseNameToDataSourcesTree;
}
CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
.getDefaultCorrelationTypes()
@ -133,13 +130,14 @@ final public class CommonAttributeCaseSearchResults {
.findFirst().get();
//Call countUniqueDataSources once to reduce the number of DB queries needed to get the frequencyPercentage
Double uniqueCaseDataSourceTuples = EamDb.getInstance().getCountUniqueDataSources().doubleValue();
Map<String, Map<String, CommonAttributeValueList>> filteredCaseNameToDataSourcesTree = new HashMap<>();
Map<String, CommonAttributeValue> valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn);
Map<String, CommonAttributeValue> valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples);
for (Entry<String, Map<String, CommonAttributeValueList>> mapOfDataSources : Collections.unmodifiableMap(metadata).entrySet()) {
if (!mapOfDataSources.getKey().equals(currentCaseName)) {
//rebuild the metadata structure with items from the current case substituted for their matches in other cases results we want to filter out removed
Map<String, CommonAttributeValueList> newTreeForCase = createTreeForCase(valuesToKeepCurrentCase, mapOfDataSources.getValue());
filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase);
if (!newTreeForCase.isEmpty()) {
filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase);
}
}
}
return filteredCaseNameToDataSourcesTree;
@ -162,21 +160,20 @@ final public class CommonAttributeCaseSearchResults {
* should not be more common than
* @param uniqueCaseDataSourceTuples the number of unique data sources in
* the CR
* @param mimeTypesToFilterOn the mimetypes to include in our results
*
* @return a map of correlation value to CommonAttributeValue for results
* from the current case
*
* @throws EamDbException
*/
private Map<String, CommonAttributeValue> getValuesToKeepFromCurrentCase(Map<String, CommonAttributeValueList> dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToFilterOn) throws EamDbException {
private Map<String, CommonAttributeValue> getValuesToKeepFromCurrentCase(Map<String, CommonAttributeValueList> dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException {
Map<String, CommonAttributeValue> valuesToKeep = new HashMap<>();
Set<String> valuesToRemove = new HashSet<>();
for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) {
if (valuesToRemove.contains(value.getValue())) {
//do nothing this value will not be added
} else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn)) {
} else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples)) {
valuesToRemove.add(value.getValue());
} else {
valuesToKeep.put(value.getValue(), value);
@ -202,7 +199,7 @@ final public class CommonAttributeCaseSearchResults {
private Map<String, CommonAttributeValueList> createTreeForCase(Map<String, CommonAttributeValue> valuesToKeepCurrentCase, Map<String, CommonAttributeValueList> dataSourceToValueList) throws EamDbException {
Map<String, CommonAttributeValueList> treeForCase = new HashMap<>();
for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) {
if (valuesToKeepCurrentCase.containsKey(value.getValue())) {
if (!treeForCase.containsKey(mapOfValueLists.getKey())) {
treeForCase.put(mapOfValueLists.getKey(), new CommonAttributeValueList());
@ -226,7 +223,6 @@ final public class CommonAttributeCaseSearchResults {
* should not be more common than
* @param uniqueCaseDataSourceTuples the number of unique data sources in
* the CR
* @param mimeTypesToInclude the mimetypes to include in our results
*
* @return true if the value should be filtered and removed from what is
* shown to the user, false if the value should not be removed and
@ -234,20 +230,7 @@ final public class CommonAttributeCaseSearchResults {
*
* @throws EamDbException
*/
private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToInclude) throws EamDbException {
//Intracase common attribute searches will have been created with an empty mimeTypesToInclude list
//because when performing intra case search this filtering will have been done during the query of the case database
if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
AbstractFile abstractFile = commonAttr.getAbstractFile();
if (abstractFile != null) {
String mimeType = abstractFile.getMIMEType();
if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
return true;
}
}
}
}
private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException {
if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
try {
Double uniqueTypeValueTuples = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(

View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -22,18 +22,15 @@ package org.sleuthkit.autopsy.commonpropertiessearch;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Stores the results from the various types of common attribute searching
@ -45,7 +42,6 @@ final public class CommonAttributeCountSearchResults {
// maps instance count to list of attribute values.
private final Map<Integer, CommonAttributeValueList> instanceCountToAttributeValues;
private final Set<String> mimeTypesToInclude;
private final int percentageThreshold;
private final int resultTypeId;
@ -58,15 +54,13 @@ final public class CommonAttributeCountSearchResults {
* common, value of 0 is disabled
* @param resultType The type of Correlation Attribute being
* searched for
* @param mimeTypesToFilterOn Set of mime types to include for intercase
* searches
*
*/
CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) {
//wrap in a new object in case any client code has used an unmodifiable collection
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
this.resultTypeId = resultType.getId();
this.mimeTypesToInclude = mimeTypesToFilterOn;
}
/**
@ -82,7 +76,6 @@ final public class CommonAttributeCountSearchResults {
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID;
this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes
}
/**
@ -152,36 +145,8 @@ final public class CommonAttributeCountSearchResults {
final Integer key = listOfValues.getKey();
final CommonAttributeValueList values = listOfValues.getValue();
for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata
//Intracase common attribute searches will have been created with an empty mimeTypesToInclude list
//because when performing intra case search this filtering will have been done during the query of the case database
boolean mimeTypeToRemove = false; //allow code to be more efficient by not attempting to remove the same value multiple times
if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
AbstractFile abstractFile = commonAttr.getAbstractFile();
if (abstractFile != null) {
String mimeType = commonAttr.getAbstractFile().getMIMEType();
if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
if (itemsToRemove.containsKey(key)) {
itemsToRemove.get(key).add(value);
} else {
List<CommonAttributeValue> toRemove = new ArrayList<>();
toRemove.add(value);
itemsToRemove.put(key, toRemove);
}
//value will be removed as the mime type existed and was not in the set to be included
//because value is removed this value does not need to be checked further
mimeTypeToRemove = true;
break;
}
}
if (mimeTypeToRemove) {
break;
}
}
}
if (!mimeTypeToRemove && maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
for (CommonAttributeValue value : values.getDelayedMetadataSet()) { // Need the real metadata
if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
try {
Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue(
attributeType, value.getValue()).doubleValue();
@ -209,7 +174,7 @@ final public class CommonAttributeCountSearchResults {
final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key);
if (instanceCountValue != null) {
instanceCountValue.removeMetaData(value);
if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata
if (instanceCountValue.getDelayedMetadataSet().isEmpty()) { // Check the real metadata
this.instanceCountToAttributeValues.remove(key);
}
}
@ -226,7 +191,7 @@ final public class CommonAttributeCountSearchResults {
int count = 0;
for (CommonAttributeValueList data : this.instanceCountToAttributeValues.values()) {
for (CommonAttributeValue md5 : data.getDelayedMetadataList()) {
for (CommonAttributeValue md5 : data.getDelayedMetadataSet()) {
count += md5.getInstanceCount();
}
}

View File

@ -1,16 +1,16 @@
/*
*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
*
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -21,7 +21,9 @@ package org.sleuthkit.autopsy.commonpropertiessearch;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* Utility and wrapper model around data required for Common Files Search
@ -34,10 +36,10 @@ final public class CommonAttributeValueList {
* The list of value nodes, which begins empty.
*/
private final List<CommonAttributeValue> metadataList;
/**
* The backing list of value nodes, which will be dynamically loaded
* when requested.
* The backing list of value nodes, which will be dynamically loaded when
* requested.
*/
private final List<CommonAttributeValue> delayedMetadataList;
@ -58,36 +60,38 @@ final public class CommonAttributeValueList {
}
/**
* Get the list of value nodes. Will be empty if
* displayDelayedMetadata() has not been called for the
* parent InstanceCountNode
* Get the list of value nodes. Will be empty if displayDelayedMetadata()
* has not been called for the parent InstanceCountNode
*
* @return metadataList the list of nodes
*/
public List<CommonAttributeValue> getMetadataList() {
return Collections.unmodifiableList(this.metadataList);
}
/**
* Get the delayed list of value nodes. Only use for
* determining how many CommonAttributeValues
* actually exist in the list.
* @return metadataList the list of nodes
* Get the delayed set of value nodes. Only use for determining which values
* and how many CommonAttributeValues actually exist in the list.
*
* @return metadataList the set of nodes
*/
List<CommonAttributeValue> getDelayedMetadataList() {
return Collections.unmodifiableList(this.delayedMetadataList);
Set<CommonAttributeValue> getDelayedMetadataSet() {
//Allows nodes to be de-duped
return new HashSet<>(this.delayedMetadataList);
}
void removeMetaData(CommonAttributeValue commonVal) {
this.delayedMetadataList.remove(commonVal);
}
/**
* Return the size of the backing list, in case
* displayDelayedMetadata() has not be called yet.
* Return the size of the backing list, in case displayDelayedMetadata() has
* not be called yet.
*
* @return int the number of matches for this value
*/
int getCommonAttributeListSize() {
return this.delayedMetadataList.size();
return this.delayedMetadataList.size();
}
/**
@ -103,6 +107,7 @@ final public class CommonAttributeValueList {
/**
* A a value node to the list, to be loaded later.
*
* @param metadata the node to add
*/
void addMetadataToList(CommonAttributeValue metadata) {

View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -110,8 +110,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode {
}
/**
* ChildFactory which builds DisplayableItem from the metadata data
* sources.
* ChildFactory which builds DisplayableItem from the metadata data sources.
*/
static class FileInstanceNodeFactory extends ChildFactory<AbstractCommonAttributeInstance> {
@ -123,7 +122,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode {
@Override
protected boolean createKeys(List<AbstractCommonAttributeInstance> list) {
for (CommonAttributeValue value : descendants.getDelayedMetadataList()) {
for (CommonAttributeValue value : descendants.getDelayedMetadataSet()) {
// This is a bit of a hack to ensure that the AbstractFile instance
// has been created before createNodesForKey() is called. Constructing
// the AbstractFile in createNodesForKey() was resulting in UI lockups.

View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -18,10 +18,16 @@
*/
package org.sleuthkit.autopsy.commonpropertiessearch;
import com.google.common.collect.Iterables;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
@ -32,12 +38,13 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil;
import org.sleuthkit.autopsy.centralrepository.datamodel.InstanceTableCallback;
import org.sleuthkit.autopsy.commonpropertiessearch.AbstractCommonAttributeInstance.NODE_TYPE;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.CaseDbAccessManager;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Used to process and return CorrelationCase values from the EamDB for
@ -45,23 +52,13 @@ import org.sleuthkit.datamodel.HashUtility;
*/
final class InterCaseSearchResultsProcessor {
private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName());
private static final String INTER_CASE_WHERE_CLAUSE = "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; //NON-NLS
/**
* The CorrelationAttributeInstance.Type this Processor will query on
*/
private final Type correlationType;
private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName());
/**
* The initial CorrelationAttributeInstance ids lookup query.
*/
private final String interCaseWhereClause;
/**
* The single CorrelationAttributeInstance object retrieval query
*/
private final String singleInterCaseWhereClause;
/**
* Used in the InterCaseCommonAttributeSearchers to find common attribute
* instances and generate nodes at the UI level.
@ -71,32 +68,6 @@ final class InterCaseSearchResultsProcessor {
*/
InterCaseSearchResultsProcessor(CorrelationAttributeInstance.Type theType) {
this.correlationType = theType;
interCaseWhereClause = getInterCaseWhereClause();
singleInterCaseWhereClause = getSingleInterCaseWhereClause();
}
private String getInterCaseWhereClause() {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType);
StringBuilder sqlString = new StringBuilder(250);
sqlString.append("value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)")
.append(" GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value");
return sqlString.toString();
}
private String getSingleInterCaseWhereClause() {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType);
StringBuilder sqlString = new StringBuilder(250);
sqlString.append("value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE value IN (SELECT value FROM ")
.append(tableName)
.append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)")
.append(" AND (case_id=%s OR case_id=%s) GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value");
return sqlString.toString();
}
/**
@ -122,30 +93,53 @@ final class InterCaseSearchResultsProcessor {
return null;
}
/**
* Get the portion of the select query which will get md5 values for files
* from the current case which are potentially being correlated on.
*
* @param mimeTypesToFilterOn the set of mime types to filter on
*
* @return the portion of a query which follows the SELECT keyword for
* finding MD5s which we are correlating on
*
* @throws EamDbException
*/
private String getFileQuery(Set<String> mimeTypesToFilterOn) throws EamDbException {
String query;
query = "md5 AS value FROM tsk_files WHERE known!=" + TskData.FileKnown.KNOWN.getFileKnownValue() + " AND md5 IS NOT NULL"; //NON-NLS
if (!mimeTypesToFilterOn.isEmpty()) {
query = query + " AND mime_type IS NOT NULL AND mime_type IN ('" + String.join("', '", mimeTypesToFilterOn) + "')"; //NON-NLS
}
return query;
}
/**
* Given the current case, fins all intercase common files from the EamDb
* and builds maps of case name to maps of data source name to
* CommonAttributeValueList.
*
* @param currentCase The current TSK Case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
*
* @return map of Case name to Maps of Datasources and their
* CommonAttributeValueLists
*/
Map<String, Map<String, CommonAttributeValueList>> findInterCaseValuesByCase(Case currentCase) {
Map<String, Map<String, CommonAttributeValueList>> findInterCaseValuesByCase(Case currentCase, Set<String> mimeTypesToFilterOn) {
try {
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -153,24 +147,30 @@ final class InterCaseSearchResultsProcessor {
/**
* Given the current case, fins all intercase common files from the EamDb
* and builds maps of obj id to md5 and case.
* and builds maps of obj id to value and case.
*
* @param currentCase The current TSK Case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
*
* @return map of number of instances to CommonAttributeValueLists
*/
Map<Integer, CommonAttributeValueList> findInterCaseValuesByCount(Case currentCase) {
Map<Integer, CommonAttributeValueList> findInterCaseValuesByCount(Case currentCase, Set<String> mimeTypesToFilterOn) {
try {
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -179,21 +179,30 @@ final class InterCaseSearchResultsProcessor {
/**
* Given the current case, and a specific case of interest, finds common
* files which exist between cases from the EamDb. Builds maps of obj id to
* md5 and case.
* value and case.
*
* @param currentCase The current TSK Case.
* @param singleCase The case of interest. Matches must exist in this case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
* @param singleCase The case of interest. Matches must exist in
* this case.
*
* @return map of number of instances to CommonAttributeValueLists
*/
Map<Integer, CommonAttributeValueList> findSingleInterCaseValuesByCount(Case currentCase, CorrelationCase singleCase) {
Map<Integer, CommonAttributeValueList> findSingleInterCaseValuesByCount(Case currentCase, Set<String> mimeTypesToFilterOn, CorrelationCase singleCase) {
try {
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
int targetCaseId = singleCase.getID();
dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback);
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId, targetCaseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -204,24 +213,30 @@ final class InterCaseSearchResultsProcessor {
* files which exist between cases from the EamDb. Builds map of case name
* to maps of data source name to CommonAttributeValueList.
*
* @param currentCase The current TSK Case.
* @param currentCase The current TSK Case.
* @param mimeTypesToFilterOn the set of mime types to filter on
* @param singleCase The case of interest. Matches must exist in
* this case.
*
* @return map of Case name to Maps of Datasources and their
* CommonAttributeValueLists
*
* @param currentCase The current TSK Case.
* @param singleCase The case of interest. Matches must exist in this case.
*/
Map<String, Map<String, CommonAttributeValueList>> findSingleInterCaseValuesByCase(Case currentCase, CorrelationCase singleCase) {
Map<String, Map<String, CommonAttributeValueList>> findSingleInterCaseValuesByCase(Case currentCase, Set<String> mimeTypesToFilterOn, CorrelationCase singleCase) {
try {
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
int targetCaseId = singleCase.getID();
dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback);
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId, targetCaseId);
if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) {
currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback);
} else {
dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
}
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
} catch (EamDbException | TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
@ -229,128 +244,146 @@ final class InterCaseSearchResultsProcessor {
/**
* Callback to use with findInterCaseValuesByCount which generates a list of
* md5s for common files search
* values for common property search
*/
private class InterCaseByCountCallback implements InstanceTableCallback {
private class InterCaseByCountCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback {
final Map<Integer, CommonAttributeValueList> instanceCollatedCommonFiles = new HashMap<>();
private final Map<Integer, CommonAttributeValueList> instanceCollatedCommonFiles = new HashMap<>();
private final int caseID;
private final int targetCase;
private CommonAttributeValue commonAttributeValue = null;
private String previousRowMd5 = "";
private InterCaseByCountCallback(int caseId) {
this(caseId, 0);
}
private InterCaseByCountCallback(int caseId, int targetCase) {
this.caseID = caseId;
this.targetCase = targetCase;
}
@Override
public void process(ResultSet resultSet) {
try {
Set<String> values = new HashSet<>();
List<Integer> targetCases = new ArrayList<>();
if (targetCase != 0) {
targetCases.add(caseID);
targetCases.add(targetCase);
}
while (resultSet.next()) {
int resultId = InstanceTableCallback.getId(resultSet);
String corValue = InstanceTableCallback.getValue(resultSet);
if (previousRowMd5.isEmpty()) {
previousRowMd5 = corValue;
}
if (corValue == null || HashUtility.isNoDataMd5(corValue)) {
continue;
}
countAndAddCommonAttributes(corValue, resultId);
values.add(corValue);
}
//Add the final instance(s)
if (commonAttributeValue != null) {
int size = commonAttributeValue.getInstanceCount();
if (instanceCollatedCommonFiles.containsKey(size)) {
instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue);
for (String corValue : values) {
List<CorrelationAttributeInstance> instances;
if (targetCases.isEmpty()) {
instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, Arrays.asList(corValue));
} else {
CommonAttributeValueList value = new CommonAttributeValueList();
value.addMetadataToList(commonAttributeValue);
instanceCollatedCommonFiles.put(size, value);
instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, Arrays.asList(corValue), targetCases);
}
int size = instances.size();
if (size > 1) {
CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue);
boolean anotherCase = false;
for (CorrelationAttributeInstance instance : instances) {
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.COUNT_NODE);
searchResult.setCurrentAttributeInst(instance);
commonAttributeValue.addInstance(searchResult);
anotherCase = anotherCase || instance.getCorrelationCase().getID() != caseID;
}
if (anotherCase) {
if (instanceCollatedCommonFiles.containsKey(size)) {
instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue);
} else {
CommonAttributeValueList value = new CommonAttributeValueList();
value.addMetadataToList(commonAttributeValue);
instanceCollatedCommonFiles.put(size, value);
}
}
}
}
} catch (SQLException ex) {
} catch (SQLException | EamDbException | CorrelationAttributeNormalizationException ex) {
LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS
}
}
/**
* Add a resultId to the list of matches for a given corValue, which
* counts to number of instances of that match, determining which
* InstanceCountNode the match will be added to.
*
* @param corValue the value which matches
* @param resultId the CorrelationAttributeInstance id to be retrieved
* later.
*/
private void countAndAddCommonAttributes(String corValue, int resultId) {
if (commonAttributeValue == null) {
commonAttributeValue = new CommonAttributeValue(corValue);
}
if (!corValue.equals(previousRowMd5)) {
int size = commonAttributeValue.getInstanceCount();
if (instanceCollatedCommonFiles.containsKey(size)) {
instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue);
} else {
CommonAttributeValueList value = new CommonAttributeValueList();
value.addMetadataToList(commonAttributeValue);
instanceCollatedCommonFiles.put(size, value);
}
commonAttributeValue = new CommonAttributeValue(corValue);
previousRowMd5 = corValue;
}
// we don't *have* all the information for the rows in the CR,
// so we need to consult the present case via the SleuthkitCase object
// Later, when the FileInstanceNode is built. Therefore, build node generators for now.
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.COUNT_NODE);
CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
searchResult.setCurrentAttributeInst(corrAttr);
commonAttributeValue.addInstance(searchResult);
}
Map<Integer, CommonAttributeValueList> getInstanceCollatedCommonFiles() {
return Collections.unmodifiableMap(instanceCollatedCommonFiles);
}
}
/**
* Callback to use with findInterCaseValuesByCount which generates a list of
* md5s for common files search
* Callback to use with findInterCaseValuesByCase which generates a map of
* maps of values for common property search
*/
private class InterCaseByCaseCallback implements InstanceTableCallback {
private class InterCaseByCaseCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback {
final Map<String, Map<String, CommonAttributeValueList>> caseCollatedDataSourceCollections = new HashMap<>();
private static final int VALUE_BATCH_SIZE = 500;
private final Map<String, Map<String, CommonAttributeValueList>> caseCollatedDataSourceCollections = new HashMap<>();
private final int caseID;
private final int targetCase;
private InterCaseByCaseCallback(int caseId) {
this(caseId, 0);
}
private InterCaseByCaseCallback(int caseId, int targetCase) {
this.caseID = caseId;
this.targetCase = targetCase;
}
@Override
public void process(ResultSet resultSet) {
try {
List<Integer> targetCases = new ArrayList<>();
if (targetCase != 0) {
targetCases.add(caseID);
targetCases.add(targetCase);
}
Set<String> values = new HashSet<>();
while (resultSet.next()) {
int resultId = InstanceTableCallback.getId(resultSet);
String corValue = InstanceTableCallback.getValue(resultSet);
if (corValue == null || HashUtility.isNoDataMd5(corValue)) {
continue;
}
CorrelationCase correlationCase = EamDb.getInstance().getCaseById(InstanceTableCallback.getCaseId(resultSet));
String caseName = correlationCase.getDisplayName();
CorrelationDataSource correlationDatasource = EamDb.getInstance().getDataSourceById(correlationCase, InstanceTableCallback.getDataSourceId(resultSet));
//label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class
String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")";
if (!caseCollatedDataSourceCollections.containsKey(caseName)) {
caseCollatedDataSourceCollections.put(caseName, new HashMap<String, CommonAttributeValueList>());
}
Map<String, CommonAttributeValueList> dataSourceToFile = caseCollatedDataSourceCollections.get(caseName);
if (!dataSourceToFile.containsKey(dataSourceNameKey)) {
dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList());
}
CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey);
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.CASE_NODE);
CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
searchResult.setCurrentAttributeInst(corrAttr);
CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue);
commonAttributeValue.addInstance(searchResult);
valueList.addMetadataToList(commonAttributeValue);
dataSourceToFile.put(dataSourceNameKey, valueList);
caseCollatedDataSourceCollections.put(caseName, dataSourceToFile);
values.add(corValue);
}
} catch (EamDbException | SQLException ex) {
for (List<String> valuesChunk : Iterables.partition(values, VALUE_BATCH_SIZE)) {
List<CorrelationAttributeInstance> instances;
if (targetCases.isEmpty()) {
instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, valuesChunk);
} else {
instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, valuesChunk, targetCases);
}
if (instances.size() > 1) {
for (CorrelationAttributeInstance instance : instances) {
CorrelationCase correlationCase = instance.getCorrelationCase();
String caseName = correlationCase.getDisplayName();
CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource();
//label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class
String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")";
if (!caseCollatedDataSourceCollections.containsKey(caseName)) {
caseCollatedDataSourceCollections.put(caseName, new HashMap<>());
}
Map<String, CommonAttributeValueList> dataSourceToFile = caseCollatedDataSourceCollections.get(caseName);
if (!dataSourceToFile.containsKey(dataSourceNameKey)) {
dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList());
}
CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey);
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE);
searchResult.setCurrentAttributeInst(instance);
CommonAttributeValue commonAttributeValue = new CommonAttributeValue(instance.getCorrelationValue());
commonAttributeValue.addInstance(searchResult);
valueList.addMetadataToList(commonAttributeValue);
dataSourceToFile.put(dataSourceNameKey, valueList);
caseCollatedDataSourceCollections.put(caseName, dataSourceToFile);
}
}
}
} catch (EamDbException | SQLException | CorrelationAttributeNormalizationException ex) {
LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS
}
}

View File

@ -2,7 +2,7 @@
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Copyright 2018-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -76,7 +76,6 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId);
this.correlationCaseName = correlationCase.getDisplayName();
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), correlationCase);
Set<String> mimeTypesToFilterOn = new HashSet<>();
if (isFilterByMedia()) {
mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
@ -84,7 +83,9 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
if (isFilterByDoc()) {
mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
}
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase);
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
/**
@ -104,7 +105,6 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId);
this.correlationCaseName = correlationCase.getDisplayName();
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), correlationCase);
Set<String> mimeTypesToFilterOn = new HashSet<>();
if (isFilterByMedia()) {
mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES);
@ -112,7 +112,9 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
if (isFilterByDoc()) {
mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES);
}
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase);
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType);
}
@NbBundle.Messages({