diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index bfcbcb6cb5..df30f3f5f8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -34,6 +34,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; import java.time.LocalDate; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -625,7 +626,7 @@ abstract class AbstractSqlEamDb implements EamDb { // This data source is already in the central repo return eamDataSource; } - + Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -650,7 +651,7 @@ abstract class AbstractSqlEamDb implements EamDb { /* * If nothing was inserted, then return the data source that * exists in the Central Repository. - * + * * This is expected to occur with PostgreSQL Central Repository * databases. */ @@ -675,7 +676,7 @@ abstract class AbstractSqlEamDb implements EamDb { * If an exception was thrown causing us to not return a new data * source, attempt to get an existing data source with the same case * ID and data source object ID. - * + * * This exception block is expected to occur with SQLite Central * Repository databases. */ @@ -1052,30 +1053,43 @@ abstract class AbstractSqlEamDb implements EamDb { } } - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - * - * @throws EamDbException - */ @Override public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { + return getArtifactInstancesByTypeValues(aType, Arrays.asList(value)); + } - String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); + @Override + public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { + return getArtifactInstances(prepareGetInstancesSql(aType, values), aType); + } - Connection conn = connect(); - - List artifactInstances = new ArrayList<>(); - - CorrelationAttributeInstance artifactInstance; - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; + @Override + public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); + String sql + = " and " + + tableName + + ".case_id in ('"; + StringBuilder inValuesBuilder = new StringBuilder(prepareGetInstancesSql(aType, values)); + inValuesBuilder.append(sql); + inValuesBuilder.append(caseIds.stream().map(String::valueOf).collect(Collectors.joining("', '"))); + inValuesBuilder.append("')"); + return getArtifactInstances(inValuesBuilder.toString(), aType); + } + /** + * Get the select statement for retrieving correlation attribute instances + * from the CR for a given type with values matching the specified values + * + * @param aType The type of the artifact + * @param values The list of correlation values to get + * CorrelationAttributeInstances for + * + * @return the select statement as a String + * + * @throws CorrelationAttributeNormalizationException + */ + private String prepareGetInstancesSql(CorrelationAttributeInstance.Type aType, List values) throws CorrelationAttributeNormalizationException { String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); String sql = "SELECT " @@ -1093,11 +1107,42 @@ abstract class AbstractSqlEamDb implements EamDb { + " LEFT JOIN data_sources ON " + tableName + ".data_source_id=data_sources.id" - + " WHERE value=?"; + + " WHERE value IN ("; + StringBuilder inValuesBuilder = new StringBuilder(sql); + for (String value : values) { + if (value != null) { + inValuesBuilder.append("'"); + inValuesBuilder.append(CorrelationAttributeNormalizer.normalize(aType, value)); + inValuesBuilder.append("',"); + } + } + inValuesBuilder.deleteCharAt(inValuesBuilder.length() - 1); //delete last comma + inValuesBuilder.append(")"); + return inValuesBuilder.toString(); + } + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValues of the given eamArtifact. + * + * @param aType The type of the artifact + * @param values The list of correlation values to get + * CorrelationAttributeInstances for + * + * @return List of artifact instances for a given type with the specified + * values + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException + */ + private List getArtifactInstances(String sql, CorrelationAttributeInstance.Type aType) throws CorrelationAttributeNormalizationException, EamDbException { + Connection conn = connect(); + List artifactInstances = new ArrayList<>(); + CorrelationAttributeInstance artifactInstance; + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; try { preparedStatement = conn.prepareStatement(sql); - preparedStatement.setString(1, normalizedValue); resultSet = preparedStatement.executeQuery(); while (resultSet.next()) { artifactInstance = getEamArtifactInstanceFromResultSet(resultSet, aType); @@ -1110,7 +1155,6 @@ abstract class AbstractSqlEamDb implements EamDb { EamDbUtil.closeResultSet(resultSet); EamDbUtil.closeConnection(conn); } - return artifactInstances; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index a5ef89caa0..1050c01ffc 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -24,7 +24,6 @@ import java.util.Set; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; -import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; /** * Main interface for interacting with the database @@ -200,27 +199,29 @@ public interface EamDb { * Creates new Data Source in the database * * @param eamDataSource the data source to add - * - * @return - A CorrelationDataSource object with data source's central repository id + * + * @return - A CorrelationDataSource object with data source's central + * repository id */ CorrelationDataSource newDataSource(CorrelationDataSource eamDataSource) throws EamDbException; - + /** * Updates the MD5 hash value in an existing data source in the database. * * @param eamDataSource The data source to update */ void updateDataSourceMd5Hash(CorrelationDataSource eamDataSource) throws EamDbException; - + /** * Updates the SHA-1 hash value in an existing data source in the database. * * @param eamDataSource The data source to update */ void updateDataSourceSha1Hash(CorrelationDataSource eamDataSource) throws EamDbException; - + /** - * Updates the SHA-256 hash value in an existing data source in the database. + * Updates the SHA-256 hash value in an existing data source in the + * database. * * @param eamDataSource The data source to update */ @@ -257,14 +258,14 @@ public interface EamDb { /** * Changes the name of a data source in the DB - * - * @param eamDataSource The data source - * @param newName The new name - * - * @throws EamDbException + * + * @param eamDataSource The data source + * @param newName The new name + * + * @throws EamDbException */ void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException; - + /** * Inserts new Artifact(s) into the database. Should add associated Case and * Data Source first. @@ -273,17 +274,55 @@ public interface EamDb { */ void addArtifactInstance(CorrelationAttributeInstance eamArtifact) throws EamDbException; + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValues of the given eamArtifact. + * + * @param aType EamArtifact.Type to search for + * @param values The list of correlation values to get + * CorrelationAttributeInstances for + * + * @return List of artifact instances for a given type with the specified + * values + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException + */ + List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException; + /** * Retrieves eamArtifact instances from the database that are associated * with the eamArtifactType and eamArtifactValue of the given eamArtifact. * - * @param aType EamArtifact.Type to search for - * @param value Value to search for + * @param aType The type of the artifact + * @param value The correlation value * * @return List of artifact instances for a given type/value + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException */ List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValues of the given eamArtifact + * for the specified cases. + * + * @param aType The type of the artifact + * @param values The list of correlation values to get + * CorrelationAttributeInstances for + * @param caseIds The list of central repository case ids to get + * CorrelationAttributeInstances for + * + * @return List of artifact instances for a given type with the specified + * values for the specified cases + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException + */ + List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException; + /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath @@ -341,7 +380,7 @@ public interface EamDb { * Retrieves number of eamArtifact instances in the database that are * associated with the given data source. * - * @param correlationDataSource Data source to search for + * @param correlationDataSource Data source to search for * * @return Number of artifact instances having caseDisplayName and * dataSource diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index a9d3b8b46d..9e486855bb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -275,7 +275,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { } @Override - public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException{ + public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException { try { acquireExclusiveLock(); super.addDataSourceObjectId(rowId, dataSourceObjectId); @@ -433,14 +433,14 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } - + /** * Changes the name of a data source in the DB - * - * @param eamDataSource The data source - * @param newName The new name - * - * @throws EamDbException + * + * @param eamDataSource The data source + * @param newName The new name + * + * @throws EamDbException */ @Override public void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException { @@ -451,7 +451,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** * Updates the MD5 hash value in an existing data source in the database. * @@ -466,7 +466,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** * Updates the SHA-1 hash value in an existing data source in the database. * @@ -481,9 +481,10 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** - * Updates the SHA-256 hash value in an existing data source in the database. + * Updates the SHA-256 hash value in an existing data source in the + * database. * * @param eamDataSource The data source to update */ @@ -513,15 +514,6 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - */ @Override public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { try { @@ -532,6 +524,26 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } + @Override + public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { + try { + acquireSharedLock(); + return super.getArtifactInstancesByTypeValues(aType, values); + } finally { + releaseSharedLock(); + } + } + + @Override + public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { + try { + acquireSharedLock(); + return super.getArtifactInstancesByTypeValuesAndCases(aType, values, caseIds); + } finally { + releaseSharedLock(); + } + } + /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java index e718c01041..ee9c70211f 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,7 +56,6 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut @Override public CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType); - Map interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase()); Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); @@ -64,13 +63,13 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn); + return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } @Override public CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType); - Map> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase()); Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); @@ -78,7 +77,8 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn); + return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } @NbBundle.Messages({ diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java index 0b61923044..3ff7d40070 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,7 +33,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNor import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; /** * Stores the results from the various types of common attribute searching @@ -55,11 +54,9 @@ final public class CommonAttributeCaseSearchResults { * common, value of 0 is disabled * @param resultType The type of Correlation Attribute being * searched for - * @param mimeTypesToFilterOn Set of mime types to include for intercase - * searches */ - CommonAttributeCaseSearchResults(Map> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set mimeTypesToFilterOn) { - this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId(), mimeTypesToFilterOn); + CommonAttributeCaseSearchResults(Map> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) { + this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId()); } /** @@ -71,7 +68,7 @@ final public class CommonAttributeCaseSearchResults { * common, value of 0 is disabled */ CommonAttributeCaseSearchResults(Map> metadata, int percentageThreshold) { - this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID, new HashSet<>()); + this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID); } /** @@ -110,11 +107,10 @@ final public class CommonAttributeCaseSearchResults { * not be more common than * @param resultTypeId the ID of the result type contained in the * metadata - * @param mimeTypesToFilterOn the mimetypes to include in our results * * @return metadata */ - private Map> filterMetadata(Map> metadata, int percentageThreshold, int resultTypeId, Set mimeTypesToFilterOn) { + private Map> filterMetadata(Map> metadata, int percentageThreshold, int resultTypeId) { try { final String currentCaseName; try { @@ -123,8 +119,9 @@ final public class CommonAttributeCaseSearchResults { throw new EamDbException("Unable to get current case while performing filtering", ex); } Map currentCaseDataSourceMap = metadata.get(currentCaseName); - if (currentCaseDataSourceMap == null) { - throw new EamDbException("No data for current case found in results, indicating there are no results and nothing will be filtered"); + Map> filteredCaseNameToDataSourcesTree = new HashMap<>(); + if (currentCaseDataSourceMap == null) { //there are no results + return filteredCaseNameToDataSourcesTree; } CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance .getDefaultCorrelationTypes() @@ -133,13 +130,14 @@ final public class CommonAttributeCaseSearchResults { .findFirst().get(); //Call countUniqueDataSources once to reduce the number of DB queries needed to get the frequencyPercentage Double uniqueCaseDataSourceTuples = EamDb.getInstance().getCountUniqueDataSources().doubleValue(); - Map> filteredCaseNameToDataSourcesTree = new HashMap<>(); - Map valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn); + Map valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples); for (Entry> mapOfDataSources : Collections.unmodifiableMap(metadata).entrySet()) { if (!mapOfDataSources.getKey().equals(currentCaseName)) { //rebuild the metadata structure with items from the current case substituted for their matches in other cases results we want to filter out removed Map newTreeForCase = createTreeForCase(valuesToKeepCurrentCase, mapOfDataSources.getValue()); - filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase); + if (!newTreeForCase.isEmpty()) { + filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase); + } } } return filteredCaseNameToDataSourcesTree; @@ -162,21 +160,20 @@ final public class CommonAttributeCaseSearchResults { * should not be more common than * @param uniqueCaseDataSourceTuples the number of unique data sources in * the CR - * @param mimeTypesToFilterOn the mimetypes to include in our results * * @return a map of correlation value to CommonAttributeValue for results * from the current case * * @throws EamDbException */ - private Map getValuesToKeepFromCurrentCase(Map dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set mimeTypesToFilterOn) throws EamDbException { + private Map getValuesToKeepFromCurrentCase(Map dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException { Map valuesToKeep = new HashMap<>(); Set valuesToRemove = new HashSet<>(); for (Entry mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) { - for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) { + for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) { if (valuesToRemove.contains(value.getValue())) { //do nothing this value will not be added - } else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn)) { + } else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples)) { valuesToRemove.add(value.getValue()); } else { valuesToKeep.put(value.getValue(), value); @@ -202,7 +199,7 @@ final public class CommonAttributeCaseSearchResults { private Map createTreeForCase(Map valuesToKeepCurrentCase, Map dataSourceToValueList) throws EamDbException { Map treeForCase = new HashMap<>(); for (Entry mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) { - for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) { + for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) { if (valuesToKeepCurrentCase.containsKey(value.getValue())) { if (!treeForCase.containsKey(mapOfValueLists.getKey())) { treeForCase.put(mapOfValueLists.getKey(), new CommonAttributeValueList()); @@ -226,7 +223,6 @@ final public class CommonAttributeCaseSearchResults { * should not be more common than * @param uniqueCaseDataSourceTuples the number of unique data sources in * the CR - * @param mimeTypesToInclude the mimetypes to include in our results * * @return true if the value should be filtered and removed from what is * shown to the user, false if the value should not be removed and @@ -234,20 +230,7 @@ final public class CommonAttributeCaseSearchResults { * * @throws EamDbException */ - private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set mimeTypesToInclude) throws EamDbException { - //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list - //because when performing intra case search this filtering will have been done during the query of the case database - if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty - for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) { - AbstractFile abstractFile = commonAttr.getAbstractFile(); - if (abstractFile != null) { - String mimeType = abstractFile.getMIMEType(); - if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) { - return true; - } - } - } - } + private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException { if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set try { Double uniqueTypeValueTuples = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue( diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java index 20751264b7..c1d2dcb481 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,18 +22,15 @@ package org.sleuthkit.autopsy.commonpropertiessearch; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; import java.util.logging.Level; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; /** * Stores the results from the various types of common attribute searching @@ -45,7 +42,6 @@ final public class CommonAttributeCountSearchResults { // maps instance count to list of attribute values. private final Map instanceCountToAttributeValues; - private final Set mimeTypesToInclude; private final int percentageThreshold; private final int resultTypeId; @@ -58,15 +54,13 @@ final public class CommonAttributeCountSearchResults { * common, value of 0 is disabled * @param resultType The type of Correlation Attribute being * searched for - * @param mimeTypesToFilterOn Set of mime types to include for intercase - * searches + * */ - CommonAttributeCountSearchResults(Map metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set mimeTypesToFilterOn) { + CommonAttributeCountSearchResults(Map metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) { //wrap in a new object in case any client code has used an unmodifiable collection this.instanceCountToAttributeValues = new HashMap<>(metadata); this.percentageThreshold = percentageThreshold; this.resultTypeId = resultType.getId(); - this.mimeTypesToInclude = mimeTypesToFilterOn; } /** @@ -82,7 +76,6 @@ final public class CommonAttributeCountSearchResults { this.instanceCountToAttributeValues = new HashMap<>(metadata); this.percentageThreshold = percentageThreshold; this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID; - this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes } /** @@ -152,36 +145,8 @@ final public class CommonAttributeCountSearchResults { final Integer key = listOfValues.getKey(); final CommonAttributeValueList values = listOfValues.getValue(); - for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata - - //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list - //because when performing intra case search this filtering will have been done during the query of the case database - boolean mimeTypeToRemove = false; //allow code to be more efficient by not attempting to remove the same value multiple times - if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty - for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) { - AbstractFile abstractFile = commonAttr.getAbstractFile(); - if (abstractFile != null) { - String mimeType = commonAttr.getAbstractFile().getMIMEType(); - if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) { - if (itemsToRemove.containsKey(key)) { - itemsToRemove.get(key).add(value); - } else { - List toRemove = new ArrayList<>(); - toRemove.add(value); - itemsToRemove.put(key, toRemove); - } - //value will be removed as the mime type existed and was not in the set to be included - //because value is removed this value does not need to be checked further - mimeTypeToRemove = true; - break; - } - } - if (mimeTypeToRemove) { - break; - } - } - } - if (!mimeTypeToRemove && maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set + for (CommonAttributeValue value : values.getDelayedMetadataSet()) { // Need the real metadata + if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set try { Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue( attributeType, value.getValue()).doubleValue(); @@ -209,7 +174,7 @@ final public class CommonAttributeCountSearchResults { final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key); if (instanceCountValue != null) { instanceCountValue.removeMetaData(value); - if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata + if (instanceCountValue.getDelayedMetadataSet().isEmpty()) { // Check the real metadata this.instanceCountToAttributeValues.remove(key); } } @@ -226,7 +191,7 @@ final public class CommonAttributeCountSearchResults { int count = 0; for (CommonAttributeValueList data : this.instanceCountToAttributeValues.values()) { - for (CommonAttributeValue md5 : data.getDelayedMetadataList()) { + for (CommonAttributeValue md5 : data.getDelayedMetadataSet()) { count += md5.getInstanceCount(); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java index 513196ed98..20ad09c797 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java @@ -1,16 +1,16 @@ /* - * + * * Autopsy Forensic Browser - * - * Copyright 2018 Basis Technology Corp. + * + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,7 +21,9 @@ package org.sleuthkit.autopsy.commonpropertiessearch; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * Utility and wrapper model around data required for Common Files Search @@ -34,10 +36,10 @@ final public class CommonAttributeValueList { * The list of value nodes, which begins empty. */ private final List metadataList; - + /** - * The backing list of value nodes, which will be dynamically loaded - * when requested. + * The backing list of value nodes, which will be dynamically loaded when + * requested. */ private final List delayedMetadataList; @@ -58,36 +60,38 @@ final public class CommonAttributeValueList { } /** - * Get the list of value nodes. Will be empty if - * displayDelayedMetadata() has not been called for the - * parent InstanceCountNode + * Get the list of value nodes. Will be empty if displayDelayedMetadata() + * has not been called for the parent InstanceCountNode + * * @return metadataList the list of nodes */ public List getMetadataList() { return Collections.unmodifiableList(this.metadataList); } - + /** - * Get the delayed list of value nodes. Only use for - * determining how many CommonAttributeValues - * actually exist in the list. - * @return metadataList the list of nodes + * Get the delayed set of value nodes. Only use for determining which values + * and how many CommonAttributeValues actually exist in the list. + * + * @return metadataList the set of nodes */ - List getDelayedMetadataList() { - return Collections.unmodifiableList(this.delayedMetadataList); + Set getDelayedMetadataSet() { + //Allows nodes to be de-duped + return new HashSet<>(this.delayedMetadataList); } - + void removeMetaData(CommonAttributeValue commonVal) { this.delayedMetadataList.remove(commonVal); } - + /** - * Return the size of the backing list, in case - * displayDelayedMetadata() has not be called yet. + * Return the size of the backing list, in case displayDelayedMetadata() has + * not be called yet. + * * @return int the number of matches for this value */ int getCommonAttributeListSize() { - return this.delayedMetadataList.size(); + return this.delayedMetadataList.size(); } /** @@ -103,6 +107,7 @@ final public class CommonAttributeValueList { /** * A a value node to the list, to be loaded later. + * * @param metadata the node to add */ void addMetadataToList(CommonAttributeValue metadata) { diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java index 8de44c415e..8451ec140d 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -110,8 +110,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode { } /** - * ChildFactory which builds DisplayableItem from the metadata data - * sources. + * ChildFactory which builds DisplayableItem from the metadata data sources. */ static class FileInstanceNodeFactory extends ChildFactory { @@ -123,7 +122,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode { @Override protected boolean createKeys(List list) { - for (CommonAttributeValue value : descendants.getDelayedMetadataList()) { + for (CommonAttributeValue value : descendants.getDelayedMetadataSet()) { // This is a bit of a hack to ensure that the AbstractFile instance // has been created before createNodesForKey() is called. Constructing // the AbstractFile in createNodesForKey() was resulting in UI lockups. diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 2c54ad65a4..52f2cec276 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,10 +18,16 @@ */ package org.sleuthkit.autopsy.commonpropertiessearch; +import com.google.common.collect.Iterables; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.List; import java.util.Map; import java.util.logging.Level; import org.sleuthkit.autopsy.casemodule.Case; @@ -32,12 +38,13 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.InstanceTableCallback; import org.sleuthkit.autopsy.commonpropertiessearch.AbstractCommonAttributeInstance.NODE_TYPE; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.CaseDbAccessManager; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.HashUtility; +import org.sleuthkit.datamodel.TskCoreException; /** * Used to process and return CorrelationCase values from the EamDB for @@ -45,23 +52,13 @@ import org.sleuthkit.datamodel.HashUtility; */ final class InterCaseSearchResultsProcessor { + private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName()); + private static final String INTER_CASE_WHERE_CLAUSE = "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; //NON-NLS /** * The CorrelationAttributeInstance.Type this Processor will query on */ private final Type correlationType; - private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName()); - - /** - * The initial CorrelationAttributeInstance ids lookup query. - */ - private final String interCaseWhereClause; - - /** - * The single CorrelationAttributeInstance object retrieval query - */ - private final String singleInterCaseWhereClause; - /** * Used in the InterCaseCommonAttributeSearchers to find common attribute * instances and generate nodes at the UI level. @@ -71,32 +68,6 @@ final class InterCaseSearchResultsProcessor { */ InterCaseSearchResultsProcessor(CorrelationAttributeInstance.Type theType) { this.correlationType = theType; - interCaseWhereClause = getInterCaseWhereClause(); - singleInterCaseWhereClause = getSingleInterCaseWhereClause(); - } - - private String getInterCaseWhereClause() { - String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType); - StringBuilder sqlString = new StringBuilder(250); - sqlString.append("value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)") - .append(" GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value"); - return sqlString.toString(); - } - - private String getSingleInterCaseWhereClause() { - String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType); - StringBuilder sqlString = new StringBuilder(250); - sqlString.append("value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)") - .append(" AND (case_id=%s OR case_id=%s) GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value"); - return sqlString.toString(); } /** @@ -122,30 +93,53 @@ final class InterCaseSearchResultsProcessor { return null; } + /** + * Get the portion of the select query which will get md5 values for files + * from the current case which are potentially being correlated on. + * + * @param mimeTypesToFilterOn the set of mime types to filter on + * + * @return the portion of a query which follows the SELECT keyword for + * finding MD5s which we are correlating on + * + * @throws EamDbException + */ + private String getFileQuery(Set mimeTypesToFilterOn) throws EamDbException { + String query; + query = "md5 AS value FROM tsk_files WHERE known!=" + TskData.FileKnown.KNOWN.getFileKnownValue() + " AND md5 IS NOT NULL"; //NON-NLS + if (!mimeTypesToFilterOn.isEmpty()) { + query = query + " AND mime_type IS NOT NULL AND mime_type IN ('" + String.join("', '", mimeTypesToFilterOn) + "')"; //NON-NLS + } + return query; + } + /** * Given the current case, fins all intercase common files from the EamDb * and builds maps of case name to maps of data source name to * CommonAttributeValueList. * - * @param currentCase The current TSK Case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on * * @return map of Case name to Maps of Datasources and their * CommonAttributeValueLists */ - Map> findInterCaseValuesByCase(Case currentCase) { + Map> findInterCaseValuesByCase(Case currentCase, Set mimeTypesToFilterOn) { try { - InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(); + EamDb dbManager = EamDb.getInstance(); - int caseId = dbManager.getCase(currentCase).getID(); - - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue()), - instancetableCallback); - + InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId); + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); } return new HashMap<>(); @@ -153,24 +147,30 @@ final class InterCaseSearchResultsProcessor { /** * Given the current case, fins all intercase common files from the EamDb - * and builds maps of obj id to md5 and case. + * and builds maps of obj id to value and case. * - * @param currentCase The current TSK Case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on + * + * @return map of number of instances to CommonAttributeValueLists */ - Map findInterCaseValuesByCount(Case currentCase) { + Map findInterCaseValuesByCount(Case currentCase, Set mimeTypesToFilterOn) { try { - InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(); + EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); - - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue()), - instancetableCallback); - + InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId); + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); } return new HashMap<>(); @@ -179,21 +179,30 @@ final class InterCaseSearchResultsProcessor { /** * Given the current case, and a specific case of interest, finds common * files which exist between cases from the EamDb. Builds maps of obj id to - * md5 and case. + * value and case. * - * @param currentCase The current TSK Case. - * @param singleCase The case of interest. Matches must exist in this case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on + * @param singleCase The case of interest. Matches must exist in + * this case. + * + * @return map of number of instances to CommonAttributeValueLists */ - Map findSingleInterCaseValuesByCount(Case currentCase, CorrelationCase singleCase) { + Map findSingleInterCaseValuesByCount(Case currentCase, Set mimeTypesToFilterOn, CorrelationCase singleCase) { try { - InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(); EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); int targetCaseId = singleCase.getID(); - dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback); + InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId, targetCaseId); + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); } return new HashMap<>(); @@ -204,24 +213,30 @@ final class InterCaseSearchResultsProcessor { * files which exist between cases from the EamDb. Builds map of case name * to maps of data source name to CommonAttributeValueList. * - * @param currentCase The current TSK Case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on + * @param singleCase The case of interest. Matches must exist in + * this case. * * @return map of Case name to Maps of Datasources and their * CommonAttributeValueLists - * - * @param currentCase The current TSK Case. - * @param singleCase The case of interest. Matches must exist in this case. */ - Map> findSingleInterCaseValuesByCase(Case currentCase, CorrelationCase singleCase) { + Map> findSingleInterCaseValuesByCase(Case currentCase, Set mimeTypesToFilterOn, CorrelationCase singleCase) { try { - InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(); + EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); int targetCaseId = singleCase.getID(); - dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback); + InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId, targetCaseId); + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); } return new HashMap<>(); @@ -229,128 +244,146 @@ final class InterCaseSearchResultsProcessor { /** * Callback to use with findInterCaseValuesByCount which generates a list of - * md5s for common files search + * values for common property search */ - private class InterCaseByCountCallback implements InstanceTableCallback { + private class InterCaseByCountCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback { - final Map instanceCollatedCommonFiles = new HashMap<>(); + private final Map instanceCollatedCommonFiles = new HashMap<>(); + private final int caseID; + private final int targetCase; - private CommonAttributeValue commonAttributeValue = null; - private String previousRowMd5 = ""; + private InterCaseByCountCallback(int caseId) { + this(caseId, 0); + } + + private InterCaseByCountCallback(int caseId, int targetCase) { + this.caseID = caseId; + this.targetCase = targetCase; + } @Override public void process(ResultSet resultSet) { try { + Set values = new HashSet<>(); + List targetCases = new ArrayList<>(); + if (targetCase != 0) { + targetCases.add(caseID); + targetCases.add(targetCase); + } while (resultSet.next()) { - - int resultId = InstanceTableCallback.getId(resultSet); String corValue = InstanceTableCallback.getValue(resultSet); - if (previousRowMd5.isEmpty()) { - previousRowMd5 = corValue; - } if (corValue == null || HashUtility.isNoDataMd5(corValue)) { continue; } - - countAndAddCommonAttributes(corValue, resultId); - + values.add(corValue); } - //Add the final instance(s) - if (commonAttributeValue != null) { - int size = commonAttributeValue.getInstanceCount(); - if (instanceCollatedCommonFiles.containsKey(size)) { - instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue); + for (String corValue : values) { + List instances; + if (targetCases.isEmpty()) { + instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, Arrays.asList(corValue)); } else { - CommonAttributeValueList value = new CommonAttributeValueList(); - value.addMetadataToList(commonAttributeValue); - instanceCollatedCommonFiles.put(size, value); + instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, Arrays.asList(corValue), targetCases); + } + int size = instances.size(); + if (size > 1) { + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); + boolean anotherCase = false; + for (CorrelationAttributeInstance instance : instances) { + CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.COUNT_NODE); + searchResult.setCurrentAttributeInst(instance); + commonAttributeValue.addInstance(searchResult); + anotherCase = anotherCase || instance.getCorrelationCase().getID() != caseID; + } + if (anotherCase) { + if (instanceCollatedCommonFiles.containsKey(size)) { + instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue); + } else { + CommonAttributeValueList value = new CommonAttributeValueList(); + value.addMetadataToList(commonAttributeValue); + instanceCollatedCommonFiles.put(size, value); + } + } } } - } catch (SQLException ex) { + } catch (SQLException | EamDbException | CorrelationAttributeNormalizationException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS } } - /** - * Add a resultId to the list of matches for a given corValue, which - * counts to number of instances of that match, determining which - * InstanceCountNode the match will be added to. - * - * @param corValue the value which matches - * @param resultId the CorrelationAttributeInstance id to be retrieved - * later. - */ - private void countAndAddCommonAttributes(String corValue, int resultId) { - if (commonAttributeValue == null) { - commonAttributeValue = new CommonAttributeValue(corValue); - } - if (!corValue.equals(previousRowMd5)) { - int size = commonAttributeValue.getInstanceCount(); - if (instanceCollatedCommonFiles.containsKey(size)) { - instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue); - } else { - CommonAttributeValueList value = new CommonAttributeValueList(); - value.addMetadataToList(commonAttributeValue); - instanceCollatedCommonFiles.put(size, value); - } - - commonAttributeValue = new CommonAttributeValue(corValue); - previousRowMd5 = corValue; - } - // we don't *have* all the information for the rows in the CR, - // so we need to consult the present case via the SleuthkitCase object - // Later, when the FileInstanceNode is built. Therefore, build node generators for now. - CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.COUNT_NODE); - CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId); - searchResult.setCurrentAttributeInst(corrAttr); - commonAttributeValue.addInstance(searchResult); - } - Map getInstanceCollatedCommonFiles() { return Collections.unmodifiableMap(instanceCollatedCommonFiles); } } /** - * Callback to use with findInterCaseValuesByCount which generates a list of - * md5s for common files search + * Callback to use with findInterCaseValuesByCase which generates a map of + * maps of values for common property search */ - private class InterCaseByCaseCallback implements InstanceTableCallback { + private class InterCaseByCaseCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback { - final Map> caseCollatedDataSourceCollections = new HashMap<>(); + private static final int VALUE_BATCH_SIZE = 500; + private final Map> caseCollatedDataSourceCollections = new HashMap<>(); + private final int caseID; + private final int targetCase; + + private InterCaseByCaseCallback(int caseId) { + this(caseId, 0); + } + + private InterCaseByCaseCallback(int caseId, int targetCase) { + this.caseID = caseId; + this.targetCase = targetCase; + } @Override public void process(ResultSet resultSet) { try { + List targetCases = new ArrayList<>(); + if (targetCase != 0) { + targetCases.add(caseID); + targetCases.add(targetCase); + } + Set values = new HashSet<>(); while (resultSet.next()) { - int resultId = InstanceTableCallback.getId(resultSet); String corValue = InstanceTableCallback.getValue(resultSet); if (corValue == null || HashUtility.isNoDataMd5(corValue)) { continue; } - CorrelationCase correlationCase = EamDb.getInstance().getCaseById(InstanceTableCallback.getCaseId(resultSet)); - String caseName = correlationCase.getDisplayName(); - CorrelationDataSource correlationDatasource = EamDb.getInstance().getDataSourceById(correlationCase, InstanceTableCallback.getDataSourceId(resultSet)); - //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class - String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; - if (!caseCollatedDataSourceCollections.containsKey(caseName)) { - caseCollatedDataSourceCollections.put(caseName, new HashMap()); - } - Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); - if (!dataSourceToFile.containsKey(dataSourceNameKey)) { - dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); - } - CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); - CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.CASE_NODE); - CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId); - searchResult.setCurrentAttributeInst(corrAttr); - CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); - commonAttributeValue.addInstance(searchResult); - valueList.addMetadataToList(commonAttributeValue); - dataSourceToFile.put(dataSourceNameKey, valueList); - caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); + values.add(corValue); } - } catch (EamDbException | SQLException ex) { + for (List valuesChunk : Iterables.partition(values, VALUE_BATCH_SIZE)) { + List instances; + if (targetCases.isEmpty()) { + instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, valuesChunk); + } else { + instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, valuesChunk, targetCases); + } + if (instances.size() > 1) { + for (CorrelationAttributeInstance instance : instances) { + CorrelationCase correlationCase = instance.getCorrelationCase(); + String caseName = correlationCase.getDisplayName(); + CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); + //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class + String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; + if (!caseCollatedDataSourceCollections.containsKey(caseName)) { + caseCollatedDataSourceCollections.put(caseName, new HashMap<>()); + } + Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); + if (!dataSourceToFile.containsKey(dataSourceNameKey)) { + dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); + } + CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); + CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); + searchResult.setCurrentAttributeInst(instance); + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(instance.getCorrelationValue()); + commonAttributeValue.addInstance(searchResult); + valueList.addMetadataToList(commonAttributeValue); + dataSourceToFile.put(dataSourceNameKey, valueList); + caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); + } + } + } + } catch (EamDbException | SQLException | CorrelationAttributeNormalizationException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java index 8129acaf16..8d0cca8412 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -76,7 +76,6 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId); this.correlationCaseName = correlationCase.getDisplayName(); InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType); - Map interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), correlationCase); Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); @@ -84,7 +83,9 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase); + + return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } /** @@ -104,7 +105,6 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId); this.correlationCaseName = correlationCase.getDisplayName(); InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType); - Map> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), correlationCase); Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); @@ -112,7 +112,9 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase); + + return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } @NbBundle.Messages({