From 13b0fe2c74d03889f87603522f1f6c4e8a93dc90 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 22 Feb 2019 16:10:58 -0500 Subject: [PATCH 01/11] 4757 changes to allow results by case search to perform quicker and hide duplicate results --- .../CommonAttributeCaseSearchResults.java | 4 +- .../CommonAttributeCountSearchResults.java | 4 +- .../CommonAttributeValueList.java | 10 +++ .../InstanceDataSourceNode.java | 2 +- .../InterCaseSearchResultsProcessor.java | 69 +++++++++++-------- 5 files changed, 54 insertions(+), 35 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java index 0b61923044..1bd94f4a8a 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java @@ -173,7 +173,7 @@ final public class CommonAttributeCaseSearchResults { Map valuesToKeep = new HashMap<>(); Set valuesToRemove = new HashSet<>(); for (Entry mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) { - for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) { + for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) { if (valuesToRemove.contains(value.getValue())) { //do nothing this value will not be added } else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn)) { @@ -202,7 +202,7 @@ final public class CommonAttributeCaseSearchResults { private Map createTreeForCase(Map valuesToKeepCurrentCase, Map dataSourceToValueList) throws EamDbException { Map treeForCase = new HashMap<>(); for (Entry mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) { - for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) { + for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) { if (valuesToKeepCurrentCase.containsKey(value.getValue())) { if (!treeForCase.containsKey(mapOfValueLists.getKey())) { treeForCase.put(mapOfValueLists.getKey(), new CommonAttributeValueList()); diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java index 20751264b7..15de0dd09a 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java @@ -152,7 +152,7 @@ final public class CommonAttributeCountSearchResults { final Integer key = listOfValues.getKey(); final CommonAttributeValueList values = listOfValues.getValue(); - for (CommonAttributeValue value : values.getDelayedMetadataList()) { // Need the real metadata + for (CommonAttributeValue value : values.getDelayedMetadataSet()) { // Need the real metadata //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list //because when performing intra case search this filtering will have been done during the query of the case database @@ -209,7 +209,7 @@ final public class CommonAttributeCountSearchResults { final CommonAttributeValueList instanceCountValue = this.instanceCountToAttributeValues.get(key); if (instanceCountValue != null) { instanceCountValue.removeMetaData(value); - if (instanceCountValue.getDelayedMetadataList().isEmpty()) { // Check the real metadata + if (instanceCountValue.getDelayedMetadataSet().isEmpty()) { // Check the real metadata this.instanceCountToAttributeValues.remove(key); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java index 513196ed98..50f29dff13 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java @@ -21,7 +21,9 @@ package org.sleuthkit.autopsy.commonpropertiessearch; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * Utility and wrapper model around data required for Common Files Search @@ -67,6 +69,10 @@ final public class CommonAttributeValueList { return Collections.unmodifiableList(this.metadataList); } + public Set getMetadataSet() { + return new HashSet<>(this.metadataList); + } + /** * Get the delayed list of value nodes. Only use for * determining how many CommonAttributeValues @@ -77,6 +83,10 @@ final public class CommonAttributeValueList { return Collections.unmodifiableList(this.delayedMetadataList); } + Set getDelayedMetadataSet() { + return new HashSet<>(this.delayedMetadataList); + } + void removeMetaData(CommonAttributeValue commonVal) { this.delayedMetadataList.remove(commonVal); } diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java index 8de44c415e..28c08e3c78 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java @@ -123,7 +123,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode { @Override protected boolean createKeys(List list) { - for (CommonAttributeValue value : descendants.getDelayedMetadataList()) { + for (CommonAttributeValue value : descendants.getDelayedMetadataSet()) { // This is a bit of a hack to ensure that the AbstractFile instance // has been created before createNodesForKey() is called. Constructing // the AbstractFile in createNodesForKey() was resulting in UI lockups. diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 2c54ad65a4..788938df5b 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -22,8 +22,12 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.List; import java.util.Map; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type; @@ -76,15 +80,7 @@ final class InterCaseSearchResultsProcessor { } private String getInterCaseWhereClause() { - String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType); - StringBuilder sqlString = new StringBuilder(250); - sqlString.append("value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)") - .append(" GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value"); - return sqlString.toString(); + return "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; } private String getSingleInterCaseWhereClause() { @@ -322,36 +318,49 @@ final class InterCaseSearchResultsProcessor { @Override public void process(ResultSet resultSet) { try { + Set values = new HashSet<>(); while (resultSet.next()) { - int resultId = InstanceTableCallback.getId(resultSet); String corValue = InstanceTableCallback.getValue(resultSet); if (corValue == null || HashUtility.isNoDataMd5(corValue)) { continue; } - CorrelationCase correlationCase = EamDb.getInstance().getCaseById(InstanceTableCallback.getCaseId(resultSet)); - String caseName = correlationCase.getDisplayName(); - CorrelationDataSource correlationDatasource = EamDb.getInstance().getDataSourceById(correlationCase, InstanceTableCallback.getDataSourceId(resultSet)); - //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class - String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; - if (!caseCollatedDataSourceCollections.containsKey(caseName)) { - caseCollatedDataSourceCollections.put(caseName, new HashMap()); + values.add(corValue); + } + for (String corValue : values){ + //select * from _instances where value=corValue && case_id!=caseId + List instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + + if (instances.size() > 1) { + for (CorrelationAttributeInstance instance : instances) { + CorrelationCase correlationCase = instance.getCorrelationCase(); + String caseName = correlationCase.getDisplayName(); + CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); + //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class + String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; + if (!caseCollatedDataSourceCollections.containsKey(caseName)) { + caseCollatedDataSourceCollections.put(caseName, new HashMap()); + } + Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); + if (!dataSourceToFile.containsKey(dataSourceNameKey)) { + dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); + } + CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); + CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); + searchResult.setCurrentAttributeInst(instance); + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); + commonAttributeValue.addInstance(searchResult); + valueList.addMetadataToList(commonAttributeValue); + dataSourceToFile.put(dataSourceNameKey, valueList); + caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); + } +// } } - Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); - if (!dataSourceToFile.containsKey(dataSourceNameKey)) { - dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); - } - CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); - CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.CASE_NODE); - CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId); - searchResult.setCurrentAttributeInst(corrAttr); - CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); - commonAttributeValue.addInstance(searchResult); - valueList.addMetadataToList(commonAttributeValue); - dataSourceToFile.put(dataSourceNameKey, valueList); - caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); + } } catch (EamDbException | SQLException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS + } catch (CorrelationAttributeNormalizationException ex) { + Exceptions.printStackTrace(ex); } } From d606d681896370b5fa9986dc7e6f2ce517ee2a41 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 22 Feb 2019 17:16:37 -0500 Subject: [PATCH 02/11] 4757 fix by count query results to be quicker --- .../InterCaseSearchResultsProcessor.java | 129 +++++++----------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 788938df5b..794d57ffec 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -231,77 +231,53 @@ final class InterCaseSearchResultsProcessor { final Map instanceCollatedCommonFiles = new HashMap<>(); - private CommonAttributeValue commonAttributeValue = null; - private String previousRowMd5 = ""; - @Override public void process(ResultSet resultSet) { try { + Set values = new HashSet<>(); + Integer caseID = null; while (resultSet.next()) { - - int resultId = InstanceTableCallback.getId(resultSet); - String corValue = InstanceTableCallback.getValue(resultSet); - if (previousRowMd5.isEmpty()) { - previousRowMd5 = corValue; + if (caseID == null) { + caseID = InstanceTableCallback.getCaseId(resultSet); } + String corValue = InstanceTableCallback.getValue(resultSet); if (corValue == null || HashUtility.isNoDataMd5(corValue)) { continue; } - - countAndAddCommonAttributes(corValue, resultId); - + values.add(corValue); } - //Add the final instance(s) - if (commonAttributeValue != null) { - int size = commonAttributeValue.getInstanceCount(); - if (instanceCollatedCommonFiles.containsKey(size)) { - instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue); - } else { - CommonAttributeValueList value = new CommonAttributeValueList(); - value.addMetadataToList(commonAttributeValue); - instanceCollatedCommonFiles.put(size, value); + for (String corValue : values) { + List instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + int size = instances.size(); + if (size > 1) { + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); + boolean anotherCase = false; + for (CorrelationAttributeInstance instance : instances) { + CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.COUNT_NODE); + searchResult.setCurrentAttributeInst(instance); + commonAttributeValue.addInstance(searchResult); + anotherCase = anotherCase || instance.getCorrelationCase().getID() != caseID; + } + if (anotherCase) { + if (instanceCollatedCommonFiles.containsKey(size)) { + instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue); + } else { + CommonAttributeValueList value = new CommonAttributeValueList(); + value.addMetadataToList(commonAttributeValue); + instanceCollatedCommonFiles.put(size, value); + } + } } } } catch (SQLException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS + } catch (EamDbException ex) { + Exceptions.printStackTrace(ex); + } catch (CorrelationAttributeNormalizationException ex) { + Exceptions.printStackTrace(ex); } } - /** - * Add a resultId to the list of matches for a given corValue, which - * counts to number of instances of that match, determining which - * InstanceCountNode the match will be added to. - * - * @param corValue the value which matches - * @param resultId the CorrelationAttributeInstance id to be retrieved - * later. - */ - private void countAndAddCommonAttributes(String corValue, int resultId) { - if (commonAttributeValue == null) { - commonAttributeValue = new CommonAttributeValue(corValue); - } - if (!corValue.equals(previousRowMd5)) { - int size = commonAttributeValue.getInstanceCount(); - if (instanceCollatedCommonFiles.containsKey(size)) { - instanceCollatedCommonFiles.get(size).addMetadataToList(commonAttributeValue); - } else { - CommonAttributeValueList value = new CommonAttributeValueList(); - value.addMetadataToList(commonAttributeValue); - instanceCollatedCommonFiles.put(size, value); - } - - commonAttributeValue = new CommonAttributeValue(corValue); - previousRowMd5 = corValue; - } - // we don't *have* all the information for the rows in the CR, - // so we need to consult the present case via the SleuthkitCase object - // Later, when the FileInstanceNode is built. Therefore, build node generators for now. - CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.COUNT_NODE); - CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId); - searchResult.setCurrentAttributeInst(corrAttr); - commonAttributeValue.addInstance(searchResult); - } - Map getInstanceCollatedCommonFiles() { return Collections.unmodifiableMap(instanceCollatedCommonFiles); } @@ -326,34 +302,31 @@ final class InterCaseSearchResultsProcessor { } values.add(corValue); } - for (String corValue : values){ - //select * from _instances where value=corValue && case_id!=caseId + for (String corValue : values) { List instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); - if (instances.size() > 1) { for (CorrelationAttributeInstance instance : instances) { CorrelationCase correlationCase = instance.getCorrelationCase(); - String caseName = correlationCase.getDisplayName(); - CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); - //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class - String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; - if (!caseCollatedDataSourceCollections.containsKey(caseName)) { - caseCollatedDataSourceCollections.put(caseName, new HashMap()); - } - Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); - if (!dataSourceToFile.containsKey(dataSourceNameKey)) { - dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); - } - CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); - CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); - searchResult.setCurrentAttributeInst(instance); - CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); - commonAttributeValue.addInstance(searchResult); - valueList.addMetadataToList(commonAttributeValue); - dataSourceToFile.put(dataSourceNameKey, valueList); - caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); + String caseName = correlationCase.getDisplayName(); + CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); + //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class + String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; + if (!caseCollatedDataSourceCollections.containsKey(caseName)) { + caseCollatedDataSourceCollections.put(caseName, new HashMap()); } -// } + Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); + if (!dataSourceToFile.containsKey(dataSourceNameKey)) { + dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); + } + CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); + CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); + searchResult.setCurrentAttributeInst(instance); + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); + commonAttributeValue.addInstance(searchResult); + valueList.addMetadataToList(commonAttributeValue); + dataSourceToFile.put(dataSourceNameKey, valueList); + caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); + } } } From b886e0b7cceff7bb5973b9584ec06852f9ec4ea4 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 25 Feb 2019 12:29:33 -0500 Subject: [PATCH 03/11] 4757 make single case queries consistant with current approach --- .../InterCaseSearchResultsProcessor.java | 112 +++++++++++------- 1 file changed, 70 insertions(+), 42 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 794d57ffec..3dafb6e06e 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -84,15 +84,7 @@ final class InterCaseSearchResultsProcessor { } private String getSingleInterCaseWhereClause() { - String tableName = EamDbUtil.correlationTypeToInstanceTableName(correlationType); - StringBuilder sqlString = new StringBuilder(250); - sqlString.append("value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE value IN (SELECT value FROM ") - .append(tableName) - .append(" WHERE case_id=%s AND (known_status !=%s OR known_status IS NULL) GROUP BY value)") - .append(" AND (case_id=%s OR case_id=%s) GROUP BY value HAVING COUNT(DISTINCT case_id) > 1) ORDER BY value"); - return sqlString.toString(); + return "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; } /** @@ -130,11 +122,11 @@ final class InterCaseSearchResultsProcessor { */ Map> findInterCaseValuesByCase(Case currentCase) { try { - InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(); + EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); - + InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId); dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); @@ -155,11 +147,11 @@ final class InterCaseSearchResultsProcessor { */ Map findInterCaseValuesByCount(Case currentCase) { try { - InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(); + EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); - + InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId); dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); @@ -182,12 +174,12 @@ final class InterCaseSearchResultsProcessor { */ Map findSingleInterCaseValuesByCount(Case currentCase, CorrelationCase singleCase) { try { - InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(); EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); int targetCaseId = singleCase.getID(); + InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId, targetCaseId); dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback); + TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); return instancetableCallback.getInstanceCollatedCommonFiles(); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); @@ -210,12 +202,14 @@ final class InterCaseSearchResultsProcessor { */ Map> findSingleInterCaseValuesByCase(Case currentCase, CorrelationCase singleCase) { try { - InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(); + EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); int targetCaseId = singleCase.getID(); + InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId, targetCaseId); dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback); + TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); + return instancetableCallback.getInstanceCollatedCommonFiles(); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); @@ -230,16 +224,23 @@ final class InterCaseSearchResultsProcessor { private class InterCaseByCountCallback implements InstanceTableCallback { final Map instanceCollatedCommonFiles = new HashMap<>(); + private final int caseID; + private final int targetCase; + + private InterCaseByCountCallback(int caseId) { + this(caseId, 0); + } + + private InterCaseByCountCallback(int caseId, int targetCase) { + this.caseID = caseId; + this.targetCase = targetCase; + } @Override public void process(ResultSet resultSet) { try { Set values = new HashSet<>(); - Integer caseID = null; while (resultSet.next()) { - if (caseID == null) { - caseID = InstanceTableCallback.getCaseId(resultSet); - } String corValue = InstanceTableCallback.getValue(resultSet); if (corValue == null || HashUtility.isNoDataMd5(corValue)) { continue; @@ -256,7 +257,9 @@ final class InterCaseSearchResultsProcessor { CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.COUNT_NODE); searchResult.setCurrentAttributeInst(instance); commonAttributeValue.addInstance(searchResult); - anotherCase = anotherCase || instance.getCorrelationCase().getID() != caseID; + if (!anotherCase && ((targetCase == 0 && instance.getCorrelationCase().getID() != caseID) || (targetCase == instance.getCorrelationCase().getID()))) { + anotherCase = true; + } } if (anotherCase) { if (instanceCollatedCommonFiles.containsKey(size)) { @@ -290,6 +293,17 @@ final class InterCaseSearchResultsProcessor { private class InterCaseByCaseCallback implements InstanceTableCallback { final Map> caseCollatedDataSourceCollections = new HashMap<>(); + private final int caseID; + private final int targetCase; + + private InterCaseByCaseCallback(int caseId) { + this(caseId, 0); + } + + private InterCaseByCaseCallback(int caseId, int targetCase) { + this.caseID = caseId; + this.targetCase = targetCase; + } @Override public void process(ResultSet resultSet) { @@ -305,30 +319,44 @@ final class InterCaseSearchResultsProcessor { for (String corValue : values) { List instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); if (instances.size() > 1) { - for (CorrelationAttributeInstance instance : instances) { - CorrelationCase correlationCase = instance.getCorrelationCase(); - String caseName = correlationCase.getDisplayName(); - CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); - //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class - String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; - if (!caseCollatedDataSourceCollections.containsKey(caseName)) { - caseCollatedDataSourceCollections.put(caseName, new HashMap()); + boolean addToResults = targetCase == 0; + if (!addToResults) { + for (CorrelationAttributeInstance instance : instances) { + if (instance.getCorrelationCase().getID() == targetCase) { + System.out.println("Target case found in results"); + addToResults = true; + break; + } } - Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); - if (!dataSourceToFile.containsKey(dataSourceNameKey)) { - dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); + } + else { + System.out.println("Target case is not set adding all results"); + } + if (addToResults) { + for (CorrelationAttributeInstance instance : instances) { + CorrelationCase correlationCase = instance.getCorrelationCase(); + String caseName = correlationCase.getDisplayName(); + CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); + //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class + String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; + if (!caseCollatedDataSourceCollections.containsKey(caseName)) { + caseCollatedDataSourceCollections.put(caseName, new HashMap<>()); + } + Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); + if (!dataSourceToFile.containsKey(dataSourceNameKey)) { + dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); + } + CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); + CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); + searchResult.setCurrentAttributeInst(instance); + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); + commonAttributeValue.addInstance(searchResult); + valueList.addMetadataToList(commonAttributeValue); + dataSourceToFile.put(dataSourceNameKey, valueList); + caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); } - CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); - CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); - searchResult.setCurrentAttributeInst(instance); - CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); - commonAttributeValue.addInstance(searchResult); - valueList.addMetadataToList(commonAttributeValue); - dataSourceToFile.put(dataSourceNameKey, valueList); - caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); } } - } } catch (EamDbException | SQLException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS From 439c43204939336199f4ea2882bbd212e166327a Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 25 Feb 2019 13:49:07 -0500 Subject: [PATCH 04/11] 4757 fix bug with target case results showing more cases --- .../datamodel/AbstractSqlEamDb.java | 138 +++++++++++++++++- .../centralrepository/datamodel/EamDb.java | 10 ++ .../InterCaseSearchResultsProcessor.java | 82 ++++++----- 3 files changed, 188 insertions(+), 42 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index bfcbcb6cb5..2987ada263 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -40,6 +40,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil.updateSchemaVersion; @@ -625,7 +626,7 @@ abstract class AbstractSqlEamDb implements EamDb { // This data source is already in the central repo return eamDataSource; } - + Connection conn = connect(); PreparedStatement preparedStatement = null; @@ -650,7 +651,7 @@ abstract class AbstractSqlEamDb implements EamDb { /* * If nothing was inserted, then return the data source that * exists in the Central Repository. - * + * * This is expected to occur with PostgreSQL Central Repository * databases. */ @@ -675,7 +676,7 @@ abstract class AbstractSqlEamDb implements EamDb { * If an exception was thrown causing us to not return a new data * source, attempt to get an existing data source with the same case * ID and data source object ID. - * + * * This exception block is expected to occur with SQLite Central * Repository databases. */ @@ -1052,6 +1053,74 @@ abstract class AbstractSqlEamDb implements EamDb { } } + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * + * @param aType The type of the artifact + * @param value The correlation value + * + * @return List of artifact instances for a given type/value + * + * @throws EamDbException + */ + public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { + + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); + String sql + = "SELECT " + + tableName + + ".id," + + tableName + + ".value," + + tableName + + ".file_obj_id," + + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, data_sources.datasource_obj_id, data_sources.md5, data_sources.sha1, data_sources.sha256 FROM " + + tableName + + " LEFT JOIN cases ON " + + tableName + + ".case_id=cases.id" + + " LEFT JOIN data_sources ON " + + tableName + + ".data_source_id=data_sources.id" + + " WHERE value IN ("; + StringBuilder inValuesBuilder = new StringBuilder(sql); + //WJS-TODO use non-stream solution to making statement for proper error handling + for (String value : values) { + if (value != null) { + inValuesBuilder.append("'"); + inValuesBuilder.append(value); + inValuesBuilder.append("',"); + } + } + inValuesBuilder.deleteCharAt(inValuesBuilder.length() - 1); //delete last comma + inValuesBuilder.append(")"); + Connection conn = connect(); + + List artifactInstances = new ArrayList<>(); + + CorrelationAttributeInstance artifactInstance; + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + + try { + preparedStatement = conn.prepareStatement(sql); + resultSet = preparedStatement.executeQuery(); + while (resultSet.next()) { + artifactInstance = getEamArtifactInstanceFromResultSet(resultSet, aType); + artifactInstances.add(artifactInstance); + } + } catch (SQLException ex) { + throw new EamDbException("Error getting artifact instances by artifactType and artifactValue.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + + return artifactInstances; + } + /** * Retrieves eamArtifact instances from the database that are associated * with the eamArtifactType and eamArtifactValue of the given eamArtifact. @@ -1114,6 +1183,69 @@ abstract class AbstractSqlEamDb implements EamDb { return artifactInstances; } + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * + * @param aType The type of the artifact + * @param value The correlation value + * + * @return List of artifact instances for a given type/value + * + * @throws EamDbException + */ + @Override + public List getArtifactInstancesByTypeValueAndCase(CorrelationAttributeInstance.Type aType, String value, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); + String sql + = "SELECT " + + tableName + + ".id," + + tableName + + ".value," + + tableName + + ".file_obj_id," + + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, data_sources.datasource_obj_id, data_sources.md5, data_sources.sha1, data_sources.sha256 FROM " + + tableName + + " LEFT JOIN cases ON " + + tableName + + ".case_id=cases.id" + + " LEFT JOIN data_sources ON " + + tableName + + ".data_source_id=data_sources.id" + + " WHERE value=? and " + + tableName + +".case_id in ('"; + StringBuilder inValuesBuilder = new StringBuilder(sql); + inValuesBuilder.append(caseIds.stream().map(String::valueOf).collect(Collectors.joining("', '"))); + inValuesBuilder.append("')"); + String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); + Connection conn = connect(); + List artifactInstances = new ArrayList<>(); + + CorrelationAttributeInstance artifactInstance; + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + + try { + preparedStatement = conn.prepareStatement(inValuesBuilder.toString()); + preparedStatement.setString(1, normalizedValue); + resultSet = preparedStatement.executeQuery(); + while (resultSet.next()) { + artifactInstance = getEamArtifactInstanceFromResultSet(resultSet, aType); + artifactInstances.add(artifactInstance); + } + } catch (SQLException ex) { + throw new EamDbException("Error getting artifact instances by artifactType and artifactValue.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + + return artifactInstances; + } + /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index a5ef89caa0..110a24e5a1 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -284,6 +284,16 @@ public interface EamDb { */ List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * + * @param aType EamArtifact.Type to search for + * @param value Value to search for + * + * @return List of artifact instances for a given type/value + */ + List getArtifactInstancesByTypeValueAndCase(CorrelationAttributeInstance.Type aType, String value, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException; /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 3dafb6e06e..e4146a15af 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.commonpropertiessearch; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -240,6 +241,11 @@ final class InterCaseSearchResultsProcessor { public void process(ResultSet resultSet) { try { Set values = new HashSet<>(); + List targetCases = new ArrayList<>(); + if (targetCase != 0) { + targetCases.add(caseID); + targetCases.add(targetCase); + } while (resultSet.next()) { String corValue = InstanceTableCallback.getValue(resultSet); if (corValue == null || HashUtility.isNoDataMd5(corValue)) { @@ -248,7 +254,12 @@ final class InterCaseSearchResultsProcessor { values.add(corValue); } for (String corValue : values) { - List instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + List instances; + if (targetCases.isEmpty()) { + instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + } else { + instances = EamDb.getInstance().getArtifactInstancesByTypeValueAndCase(correlationType, corValue, targetCases); + } int size = instances.size(); if (size > 1) { CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); @@ -257,9 +268,7 @@ final class InterCaseSearchResultsProcessor { CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.COUNT_NODE); searchResult.setCurrentAttributeInst(instance); commonAttributeValue.addInstance(searchResult); - if (!anotherCase && ((targetCase == 0 && instance.getCorrelationCase().getID() != caseID) || (targetCase == instance.getCorrelationCase().getID()))) { - anotherCase = true; - } + anotherCase = anotherCase || instance.getCorrelationCase().getID() != caseID; } if (anotherCase) { if (instanceCollatedCommonFiles.containsKey(size)) { @@ -308,6 +317,11 @@ final class InterCaseSearchResultsProcessor { @Override public void process(ResultSet resultSet) { try { + List targetCases = new ArrayList<>(); + if (targetCase != 0) { + targetCases.add(caseID); + targetCases.add(targetCase); + } Set values = new HashSet<>(); while (resultSet.next()) { String corValue = InstanceTableCallback.getValue(resultSet); @@ -317,44 +331,34 @@ final class InterCaseSearchResultsProcessor { values.add(corValue); } for (String corValue : values) { - List instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + List instances; + if (targetCases.isEmpty()) { + instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + } else { + instances = EamDb.getInstance().getArtifactInstancesByTypeValueAndCase(correlationType, corValue, targetCases); + } if (instances.size() > 1) { - boolean addToResults = targetCase == 0; - if (!addToResults) { - for (CorrelationAttributeInstance instance : instances) { - if (instance.getCorrelationCase().getID() == targetCase) { - System.out.println("Target case found in results"); - addToResults = true; - break; - } + for (CorrelationAttributeInstance instance : instances) { + CorrelationCase correlationCase = instance.getCorrelationCase(); + String caseName = correlationCase.getDisplayName(); + CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); + //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class + String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; + if (!caseCollatedDataSourceCollections.containsKey(caseName)) { + caseCollatedDataSourceCollections.put(caseName, new HashMap<>()); } - } - else { - System.out.println("Target case is not set adding all results"); - } - if (addToResults) { - for (CorrelationAttributeInstance instance : instances) { - CorrelationCase correlationCase = instance.getCorrelationCase(); - String caseName = correlationCase.getDisplayName(); - CorrelationDataSource correlationDatasource = instance.getCorrelationDataSource(); - //label datasource with it's id for uniqueness done in same manner as ImageGallery does in the DataSourceCell class - String dataSourceNameKey = correlationDatasource.getName() + " (Id: " + correlationDatasource.getDataSourceObjectID() + ")"; - if (!caseCollatedDataSourceCollections.containsKey(caseName)) { - caseCollatedDataSourceCollections.put(caseName, new HashMap<>()); - } - Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); - if (!dataSourceToFile.containsKey(dataSourceNameKey)) { - dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); - } - CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); - CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); - searchResult.setCurrentAttributeInst(instance); - CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); - commonAttributeValue.addInstance(searchResult); - valueList.addMetadataToList(commonAttributeValue); - dataSourceToFile.put(dataSourceNameKey, valueList); - caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); + Map dataSourceToFile = caseCollatedDataSourceCollections.get(caseName); + if (!dataSourceToFile.containsKey(dataSourceNameKey)) { + dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList()); } + CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); + CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); + searchResult.setCurrentAttributeInst(instance); + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); + commonAttributeValue.addInstance(searchResult); + valueList.addMetadataToList(commonAttributeValue); + dataSourceToFile.put(dataSourceNameKey, valueList); + caseCollatedDataSourceCollections.put(caseName, dataSourceToFile); } } } From aac9ced6123e7a42e61a9fb2ab5e2a6a644bb1d2 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 25 Feb 2019 16:37:24 -0500 Subject: [PATCH 05/11] 4757 combine queries made for by case results --- .../datamodel/AbstractSqlEamDb.java | 144 ++++-------------- .../centralrepository/datamodel/EamDb.java | 15 +- .../datamodel/SqliteEamDb.java | 40 ++++- .../InterCaseSearchResultsProcessor.java | 16 +- 4 files changed, 93 insertions(+), 122 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 2987ada263..26af02e7e2 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -34,6 +34,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; import java.time.LocalDate; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -1064,8 +1065,28 @@ abstract class AbstractSqlEamDb implements EamDb { * * @throws EamDbException */ + @Override + public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { + return getArtifactInstancesByTypeValues(aType, Arrays.asList(value)); + } + + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * + * @param aType The type of the artifact + * @param value The correlation value + * + * @return List of artifact instances for a given type/value + * + * @throws EamDbException + */ + @Override public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { + return getArtifactInstances(prepareGetInstancesSql(aType, values), aType); + } + private String prepareGetInstancesSql(CorrelationAttributeInstance.Type aType, List values) throws CorrelationAttributeNormalizationException { String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); String sql = "SELECT " @@ -1085,24 +1106,24 @@ abstract class AbstractSqlEamDb implements EamDb { + ".data_source_id=data_sources.id" + " WHERE value IN ("; StringBuilder inValuesBuilder = new StringBuilder(sql); - //WJS-TODO use non-stream solution to making statement for proper error handling for (String value : values) { if (value != null) { inValuesBuilder.append("'"); - inValuesBuilder.append(value); + inValuesBuilder.append(CorrelationAttributeNormalizer.normalize(aType, value)); inValuesBuilder.append("',"); } } inValuesBuilder.deleteCharAt(inValuesBuilder.length() - 1); //delete last comma inValuesBuilder.append(")"); + return inValuesBuilder.toString(); + } + + private List getArtifactInstances(String sql, CorrelationAttributeInstance.Type aType) throws CorrelationAttributeNormalizationException, EamDbException { Connection conn = connect(); - List artifactInstances = new ArrayList<>(); - CorrelationAttributeInstance artifactInstance; PreparedStatement preparedStatement = null; ResultSet resultSet = null; - try { preparedStatement = conn.prepareStatement(sql); resultSet = preparedStatement.executeQuery(); @@ -1117,7 +1138,6 @@ abstract class AbstractSqlEamDb implements EamDb { EamDbUtil.closeResultSet(resultSet); EamDbUtil.closeConnection(conn); } - return artifactInstances; } @@ -1133,117 +1153,17 @@ abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { - - String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - - Connection conn = connect(); - - List artifactInstances = new ArrayList<>(); - - CorrelationAttributeInstance artifactInstance; - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; - + public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); String sql - = "SELECT " + = " and " + tableName - + ".id," - + tableName - + ".value," - + tableName - + ".file_obj_id," - + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, data_sources.datasource_obj_id, data_sources.md5, data_sources.sha1, data_sources.sha256 FROM " - + tableName - + " LEFT JOIN cases ON " - + tableName - + ".case_id=cases.id" - + " LEFT JOIN data_sources ON " - + tableName - + ".data_source_id=data_sources.id" - + " WHERE value=?"; - - try { - preparedStatement = conn.prepareStatement(sql); - preparedStatement.setString(1, normalizedValue); - resultSet = preparedStatement.executeQuery(); - while (resultSet.next()) { - artifactInstance = getEamArtifactInstanceFromResultSet(resultSet, aType); - artifactInstances.add(artifactInstance); - } - } catch (SQLException ex) { - throw new EamDbException("Error getting artifact instances by artifactType and artifactValue.", ex); // NON-NLS - } finally { - EamDbUtil.closeStatement(preparedStatement); - EamDbUtil.closeResultSet(resultSet); - EamDbUtil.closeConnection(conn); - } - - return artifactInstances; - } - - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - * - * @throws EamDbException - */ - @Override - public List getArtifactInstancesByTypeValueAndCase(CorrelationAttributeInstance.Type aType, String value, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { - String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); - String sql - = "SELECT " - + tableName - + ".id," - + tableName - + ".value," - + tableName - + ".file_obj_id," - + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, data_sources.datasource_obj_id, data_sources.md5, data_sources.sha1, data_sources.sha256 FROM " - + tableName - + " LEFT JOIN cases ON " - + tableName - + ".case_id=cases.id" - + " LEFT JOIN data_sources ON " - + tableName - + ".data_source_id=data_sources.id" - + " WHERE value=? and " - + tableName - +".case_id in ('"; - StringBuilder inValuesBuilder = new StringBuilder(sql); + + ".case_id in ('"; + StringBuilder inValuesBuilder = new StringBuilder(prepareGetInstancesSql(aType, values)); + inValuesBuilder.append(sql); inValuesBuilder.append(caseIds.stream().map(String::valueOf).collect(Collectors.joining("', '"))); inValuesBuilder.append("')"); - String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - Connection conn = connect(); - List artifactInstances = new ArrayList<>(); - - CorrelationAttributeInstance artifactInstance; - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; - - try { - preparedStatement = conn.prepareStatement(inValuesBuilder.toString()); - preparedStatement.setString(1, normalizedValue); - resultSet = preparedStatement.executeQuery(); - while (resultSet.next()) { - artifactInstance = getEamArtifactInstanceFromResultSet(resultSet, aType); - artifactInstances.add(artifactInstance); - } - } catch (SQLException ex) { - throw new EamDbException("Error getting artifact instances by artifactType and artifactValue.", ex); // NON-NLS - } finally { - EamDbUtil.closeStatement(preparedStatement); - EamDbUtil.closeResultSet(resultSet); - EamDbUtil.closeConnection(conn); - } - - return artifactInstances; + return getArtifactInstances(inValuesBuilder.toString(), aType); } /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index 110a24e5a1..ce5052d0a9 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -282,7 +282,7 @@ public interface EamDb { * * @return List of artifact instances for a given type/value */ - List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; + List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException; /** * Retrieves eamArtifact instances from the database that are associated @@ -293,7 +293,18 @@ public interface EamDb { * * @return List of artifact instances for a given type/value */ - List getArtifactInstancesByTypeValueAndCase(CorrelationAttributeInstance.Type aType, String value, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException; + List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; + + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * + * @param aType EamArtifact.Type to search for + * @param value Value to search for + * + * @return List of artifact instances for a given type/value + */ + List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException; /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index a9d3b8b46d..aa02a57839 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -512,7 +512,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** * Retrieves eamArtifact instances from the database that are associated * with the eamArtifactType and eamArtifactValue of the given eamArtifact. @@ -531,7 +531,45 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } + + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * + * @param aType The type of the artifact + * @param value The correlation value + * + * @return List of artifact instances for a given type/value + */ + @Override + public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { + try { + acquireSharedLock(); + return super.getArtifactInstancesByTypeValues(aType, values); + } finally { + releaseSharedLock(); + } + } + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * + * @param aType The type of the artifact + * @param value The correlation value + * + * @return List of artifact instances for a given type/value + */ + @Override + public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { + try { + acquireSharedLock(); + return super.getArtifactInstancesByTypeValuesAndCases(aType, values, caseIds); + } finally { + releaseSharedLock(); + } + } + /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index e4146a15af..2b4677cfe4 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -18,9 +18,11 @@ */ package org.sleuthkit.autopsy.commonpropertiessearch; +import com.google.common.collect.Iterables; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -37,7 +39,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.InstanceTableCallback; import org.sleuthkit.autopsy.commonpropertiessearch.AbstractCommonAttributeInstance.NODE_TYPE; import org.sleuthkit.autopsy.coreutils.Logger; @@ -256,9 +257,9 @@ final class InterCaseSearchResultsProcessor { for (String corValue : values) { List instances; if (targetCases.isEmpty()) { - instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, Arrays.asList(corValue)); } else { - instances = EamDb.getInstance().getArtifactInstancesByTypeValueAndCase(correlationType, corValue, targetCases); + instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, Arrays.asList(corValue), targetCases); } int size = instances.size(); if (size > 1) { @@ -301,6 +302,7 @@ final class InterCaseSearchResultsProcessor { */ private class InterCaseByCaseCallback implements InstanceTableCallback { + private static final int VALUE_BATCH_SIZE = 500; final Map> caseCollatedDataSourceCollections = new HashMap<>(); private final int caseID; private final int targetCase; @@ -330,12 +332,12 @@ final class InterCaseSearchResultsProcessor { } values.add(corValue); } - for (String corValue : values) { + for (List valuesChunk : Iterables.partition(values, VALUE_BATCH_SIZE)) { List instances; if (targetCases.isEmpty()) { - instances = EamDb.getInstance().getArtifactInstancesByTypeValue(correlationType, corValue); + instances = EamDb.getInstance().getArtifactInstancesByTypeValues(correlationType, valuesChunk); } else { - instances = EamDb.getInstance().getArtifactInstancesByTypeValueAndCase(correlationType, corValue, targetCases); + instances = EamDb.getInstance().getArtifactInstancesByTypeValuesAndCases(correlationType, valuesChunk, targetCases); } if (instances.size() > 1) { for (CorrelationAttributeInstance instance : instances) { @@ -354,7 +356,7 @@ final class InterCaseSearchResultsProcessor { CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey); CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(instance.getID(), correlationType, NODE_TYPE.CASE_NODE); searchResult.setCurrentAttributeInst(instance); - CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue); + CommonAttributeValue commonAttributeValue = new CommonAttributeValue(instance.getCorrelationValue()); commonAttributeValue.addInstance(searchResult); valueList.addMetadataToList(commonAttributeValue); dataSourceToFile.put(dataSourceNameKey, valueList); From 9009550a8b25d7dc2d7af197230820026aa225f9 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 26 Feb 2019 16:26:17 -0500 Subject: [PATCH 06/11] 4757 perform mime type filtering as part of file hash query --- .../AllInterCaseCommonAttributeSearcher.java | 9 +- .../CommonAttributeCaseSearchResults.java | 32 ++----- .../CommonAttributeCountSearchResults.java | 38 +------- .../InterCaseSearchResultsProcessor.java | 89 +++++++++++++------ ...ingleInterCaseCommonAttributeSearcher.java | 12 +-- 5 files changed, 86 insertions(+), 94 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java index e718c01041..e9088dbbb6 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java @@ -56,7 +56,6 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut @Override public CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType); - Map interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase()); Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); @@ -64,13 +63,14 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn); + + return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } @Override public CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType); - Map> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase()); Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); @@ -78,7 +78,8 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn); + return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } @NbBundle.Messages({ diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java index 1bd94f4a8a..317c5a26ab 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java @@ -33,7 +33,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNor import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; /** * Stores the results from the various types of common attribute searching @@ -55,11 +54,9 @@ final public class CommonAttributeCaseSearchResults { * common, value of 0 is disabled * @param resultType The type of Correlation Attribute being * searched for - * @param mimeTypesToFilterOn Set of mime types to include for intercase - * searches */ - CommonAttributeCaseSearchResults(Map> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set mimeTypesToFilterOn) { - this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId(), mimeTypesToFilterOn); + CommonAttributeCaseSearchResults(Map> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) { + this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId()); } /** @@ -71,7 +68,7 @@ final public class CommonAttributeCaseSearchResults { * common, value of 0 is disabled */ CommonAttributeCaseSearchResults(Map> metadata, int percentageThreshold) { - this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID, new HashSet<>()); + this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID); } /** @@ -114,7 +111,7 @@ final public class CommonAttributeCaseSearchResults { * * @return metadata */ - private Map> filterMetadata(Map> metadata, int percentageThreshold, int resultTypeId, Set mimeTypesToFilterOn) { + private Map> filterMetadata(Map> metadata, int percentageThreshold, int resultTypeId) { try { final String currentCaseName; try { @@ -134,7 +131,7 @@ final public class CommonAttributeCaseSearchResults { //Call countUniqueDataSources once to reduce the number of DB queries needed to get the frequencyPercentage Double uniqueCaseDataSourceTuples = EamDb.getInstance().getCountUniqueDataSources().doubleValue(); Map> filteredCaseNameToDataSourcesTree = new HashMap<>(); - Map valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn); + Map valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples); for (Entry> mapOfDataSources : Collections.unmodifiableMap(metadata).entrySet()) { if (!mapOfDataSources.getKey().equals(currentCaseName)) { //rebuild the metadata structure with items from the current case substituted for their matches in other cases results we want to filter out removed @@ -169,14 +166,14 @@ final public class CommonAttributeCaseSearchResults { * * @throws EamDbException */ - private Map getValuesToKeepFromCurrentCase(Map dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set mimeTypesToFilterOn) throws EamDbException { + private Map getValuesToKeepFromCurrentCase(Map dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException { Map valuesToKeep = new HashMap<>(); Set valuesToRemove = new HashSet<>(); for (Entry mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) { for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataSet()) { if (valuesToRemove.contains(value.getValue())) { //do nothing this value will not be added - } else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn)) { + } else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples)) { valuesToRemove.add(value.getValue()); } else { valuesToKeep.put(value.getValue(), value); @@ -234,20 +231,7 @@ final public class CommonAttributeCaseSearchResults { * * @throws EamDbException */ - private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set mimeTypesToInclude) throws EamDbException { - //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list - //because when performing intra case search this filtering will have been done during the query of the case database - if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty - for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) { - AbstractFile abstractFile = commonAttr.getAbstractFile(); - if (abstractFile != null) { - String mimeType = abstractFile.getMIMEType(); - if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) { - return true; - } - } - } - } + private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples) throws EamDbException { if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set try { Double uniqueTypeValueTuples = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue( diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java index 15de0dd09a..4845218680 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java @@ -45,7 +45,6 @@ final public class CommonAttributeCountSearchResults { // maps instance count to list of attribute values. private final Map instanceCountToAttributeValues; - private final Set mimeTypesToInclude; private final int percentageThreshold; private final int resultTypeId; @@ -58,15 +57,13 @@ final public class CommonAttributeCountSearchResults { * common, value of 0 is disabled * @param resultType The type of Correlation Attribute being * searched for - * @param mimeTypesToFilterOn Set of mime types to include for intercase - * searches + */ - CommonAttributeCountSearchResults(Map metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set mimeTypesToFilterOn) { + CommonAttributeCountSearchResults(Map metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) { //wrap in a new object in case any client code has used an unmodifiable collection this.instanceCountToAttributeValues = new HashMap<>(metadata); this.percentageThreshold = percentageThreshold; this.resultTypeId = resultType.getId(); - this.mimeTypesToInclude = mimeTypesToFilterOn; } /** @@ -82,7 +79,6 @@ final public class CommonAttributeCountSearchResults { this.instanceCountToAttributeValues = new HashMap<>(metadata); this.percentageThreshold = percentageThreshold; this.resultTypeId = CorrelationAttributeInstance.FILES_TYPE_ID; - this.mimeTypesToInclude = new HashSet<>(); //don't filter on mimetypes } /** @@ -153,35 +149,7 @@ final public class CommonAttributeCountSearchResults { final CommonAttributeValueList values = listOfValues.getValue(); for (CommonAttributeValue value : values.getDelayedMetadataSet()) { // Need the real metadata - - //Intracase common attribute searches will have been created with an empty mimeTypesToInclude list - //because when performing intra case search this filtering will have been done during the query of the case database - boolean mimeTypeToRemove = false; //allow code to be more efficient by not attempting to remove the same value multiple times - if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty - for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) { - AbstractFile abstractFile = commonAttr.getAbstractFile(); - if (abstractFile != null) { - String mimeType = commonAttr.getAbstractFile().getMIMEType(); - if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) { - if (itemsToRemove.containsKey(key)) { - itemsToRemove.get(key).add(value); - } else { - List toRemove = new ArrayList<>(); - toRemove.add(value); - itemsToRemove.put(key, toRemove); - } - //value will be removed as the mime type existed and was not in the set to be included - //because value is removed this value does not need to be checked further - mimeTypeToRemove = true; - break; - } - } - if (mimeTypeToRemove) { - break; - } - } - } - if (!mimeTypeToRemove && maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set + if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set try { Double uniqueTypeValueTuples = eamDb.getCountUniqueCaseDataSourceTuplesHavingTypeValue( attributeType, value.getValue()).doubleValue(); diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 2b4677cfe4..9b3500971f 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -30,6 +30,8 @@ import java.util.Set; import java.util.List; import java.util.Map; import java.util.logging.Level; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; @@ -42,8 +44,13 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.centralrepository.datamodel.InstanceTableCallback; import org.sleuthkit.autopsy.commonpropertiessearch.AbstractCommonAttributeInstance.NODE_TYPE; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.CaseDbAccessManager; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.HashUtility; +import org.sleuthkit.datamodel.TskCoreException; /** * Used to process and return CorrelationCase values from the EamDB for @@ -51,13 +58,12 @@ import org.sleuthkit.datamodel.HashUtility; */ final class InterCaseSearchResultsProcessor { + private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName()); /** * The CorrelationAttributeInstance.Type this Processor will query on */ private final Type correlationType; - private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName()); - /** * The initial CorrelationAttributeInstance ids lookup query. */ @@ -112,6 +118,15 @@ final class InterCaseSearchResultsProcessor { return null; } + private String getFileQuery(Set mimeTypesToFilterOn) throws EamDbException { + String query; + query = "md5 as value from tsk_files where known!=" + TskData.FileKnown.KNOWN.getFileKnownValue() + " AND md5 IS NOT NULL"; + if (!mimeTypesToFilterOn.isEmpty()) { + query = query + " AND mime_type IS NOT NULL AND mime_type IN ('" + String.join("', '", mimeTypesToFilterOn) + "')"; + } + return query; + } + /** * Given the current case, fins all intercase common files from the EamDb * and builds maps of case name to maps of data source name to @@ -122,21 +137,25 @@ final class InterCaseSearchResultsProcessor { * @return map of Case name to Maps of Datasources and their * CommonAttributeValueLists */ - Map> findInterCaseValuesByCase(Case currentCase) { + Map> findInterCaseValuesByCase(Case currentCase, Set mimeTypesToFilterOn) { try { EamDb dbManager = EamDb.getInstance(); - int caseId = dbManager.getCase(currentCase).getID(); InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId); - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue()), - instancetableCallback); - + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); } return new HashMap<>(); } @@ -147,21 +166,26 @@ final class InterCaseSearchResultsProcessor { * * @param currentCase The current TSK Case. */ - Map findInterCaseValuesByCount(Case currentCase) { + Map findInterCaseValuesByCount(Case currentCase, Set mimeTypesToFilterOn) { try { EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId); - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue()), - instancetableCallback); - + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); } return new HashMap<>(); } @@ -174,17 +198,24 @@ final class InterCaseSearchResultsProcessor { * @param currentCase The current TSK Case. * @param singleCase The case of interest. Matches must exist in this case. */ - Map findSingleInterCaseValuesByCount(Case currentCase, CorrelationCase singleCase) { + Map findSingleInterCaseValuesByCount(Case currentCase, Set mimeTypesToFilterOn, CorrelationCase singleCase) { try { EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); int targetCaseId = singleCase.getID(); InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback(caseId, targetCaseId); - dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); } return new HashMap<>(); } @@ -202,30 +233,36 @@ final class InterCaseSearchResultsProcessor { * @param currentCase The current TSK Case. * @param singleCase The case of interest. Matches must exist in this case. */ - Map> findSingleInterCaseValuesByCase(Case currentCase, CorrelationCase singleCase) { + Map> findSingleInterCaseValuesByCase(Case currentCase, Set mimeTypesToFilterOn, CorrelationCase singleCase) { try { EamDb dbManager = EamDb.getInstance(); int caseId = dbManager.getCase(currentCase).getID(); int targetCaseId = singleCase.getID(); InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback(caseId, targetCaseId); - dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId, - TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); - + if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); + } else { + dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + TskData.FileKnown.KNOWN.getFileKnownValue()), + instancetableCallback); + } return instancetableCallback.getInstanceCollatedCommonFiles(); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); } return new HashMap<>(); } /** * Callback to use with findInterCaseValuesByCount which generates a list of - * md5s for common files search + * values for common property search */ - private class InterCaseByCountCallback implements InstanceTableCallback { + private class InterCaseByCountCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback { - final Map instanceCollatedCommonFiles = new HashMap<>(); + private final Map instanceCollatedCommonFiles = new HashMap<>(); private final int caseID; private final int targetCase; @@ -298,12 +335,12 @@ final class InterCaseSearchResultsProcessor { /** * Callback to use with findInterCaseValuesByCount which generates a list of - * md5s for common files search + * values for common property search */ - private class InterCaseByCaseCallback implements InstanceTableCallback { + private class InterCaseByCaseCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback { private static final int VALUE_BATCH_SIZE = 500; - final Map> caseCollatedDataSourceCollections = new HashMap<>(); + private final Map> caseCollatedDataSourceCollections = new HashMap<>(); private final int caseID; private final int targetCase; diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java index 8129acaf16..e098c1ea3d 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java @@ -76,15 +76,16 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId); this.correlationCaseName = correlationCase.getDisplayName(); InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType); - Map interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), correlationCase); - Set mimeTypesToFilterOn = new HashSet<>(); + Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); } if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase); + + return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } /** @@ -104,7 +105,6 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId); this.correlationCaseName = correlationCase.getDisplayName(); InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType); - Map> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), correlationCase); Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); @@ -112,7 +112,9 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri if (isFilterByDoc()) { mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } - return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn); + Map> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), mimeTypesToFilterOn, correlationCase); + + return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } @NbBundle.Messages({ From 5e7fe17d072acf46c84e689fade9b10117bb7f2d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 26 Feb 2019 17:33:59 -0500 Subject: [PATCH 07/11] 4757 first pass at clean up for common property search changes --- .../datamodel/AbstractSqlEamDb.java | 93 +++++++------- .../centralrepository/datamodel/EamDb.java | 68 +++++++---- .../datamodel/SqliteEamDb.java | 27 ----- .../AllInterCaseCommonAttributeSearcher.java | 3 +- .../CommonAttributeCaseSearchResults.java | 7 +- .../CommonAttributeCountSearchResults.java | 7 +- .../CommonAttributeValueList.java | 57 ++++----- .../InstanceDataSourceNode.java | 2 +- .../InterCaseSearchResultsProcessor.java | 114 ++++++++---------- ...ingleInterCaseCommonAttributeSearcher.java | 4 +- 10 files changed, 167 insertions(+), 215 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 26af02e7e2..520908f125 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -41,7 +41,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; -import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil.updateSchemaVersion; @@ -1054,38 +1053,43 @@ abstract class AbstractSqlEamDb implements EamDb { } } - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - * - * @throws EamDbException - */ @Override public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { return getArtifactInstancesByTypeValues(aType, Arrays.asList(value)); } - - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - * - * @throws EamDbException - */ + + @Override public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { return getArtifactInstances(prepareGetInstancesSql(aType, values), aType); } - + + @Override + public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); + String sql + = " and " + + tableName + + ".case_id in ('"; + StringBuilder inValuesBuilder = new StringBuilder(prepareGetInstancesSql(aType, values)); + inValuesBuilder.append(sql); + inValuesBuilder.append(caseIds.stream().map(String::valueOf).collect(Collectors.joining("', '"))); + inValuesBuilder.append("')"); + return getArtifactInstances(inValuesBuilder.toString(), aType); + } + + /** + * Get the select statement for retrieving correlation attribute instances + * from the CR for a given type with values matching the specified values + * + * @param aType The type of the artifact + * @param values The list of correlation values to get + * CorrelationAttributeInstances for + * + * @return the select statement as a String + * + * @throws CorrelationAttributeNormalizationException + */ private String prepareGetInstancesSql(CorrelationAttributeInstance.Type aType, List values) throws CorrelationAttributeNormalizationException { String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); String sql @@ -1118,6 +1122,20 @@ abstract class AbstractSqlEamDb implements EamDb { return inValuesBuilder.toString(); } + /** + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValues of the given eamArtifact. + * + * @param aType The type of the artifact + * @param values The list of correlation values to get + * CorrelationAttributeInstances for + * + * @return List of artifact instances for a given type with the specified + * values + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException + */ private List getArtifactInstances(String sql, CorrelationAttributeInstance.Type aType) throws CorrelationAttributeNormalizationException, EamDbException { Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -1141,31 +1159,6 @@ abstract class AbstractSqlEamDb implements EamDb { return artifactInstances; } - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - * - * @throws EamDbException - */ - @Override - public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { - String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); - String sql - = " and " - + tableName - + ".case_id in ('"; - StringBuilder inValuesBuilder = new StringBuilder(prepareGetInstancesSql(aType, values)); - inValuesBuilder.append(sql); - inValuesBuilder.append(caseIds.stream().map(String::valueOf).collect(Collectors.joining("', '"))); - inValuesBuilder.append("')"); - return getArtifactInstances(inValuesBuilder.toString(), aType); - } - /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index ce5052d0a9..1050c01ffc 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -24,7 +24,6 @@ import java.util.Set; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; -import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; /** * Main interface for interacting with the database @@ -200,27 +199,29 @@ public interface EamDb { * Creates new Data Source in the database * * @param eamDataSource the data source to add - * - * @return - A CorrelationDataSource object with data source's central repository id + * + * @return - A CorrelationDataSource object with data source's central + * repository id */ CorrelationDataSource newDataSource(CorrelationDataSource eamDataSource) throws EamDbException; - + /** * Updates the MD5 hash value in an existing data source in the database. * * @param eamDataSource The data source to update */ void updateDataSourceMd5Hash(CorrelationDataSource eamDataSource) throws EamDbException; - + /** * Updates the SHA-1 hash value in an existing data source in the database. * * @param eamDataSource The data source to update */ void updateDataSourceSha1Hash(CorrelationDataSource eamDataSource) throws EamDbException; - + /** - * Updates the SHA-256 hash value in an existing data source in the database. + * Updates the SHA-256 hash value in an existing data source in the + * database. * * @param eamDataSource The data source to update */ @@ -257,14 +258,14 @@ public interface EamDb { /** * Changes the name of a data source in the DB - * - * @param eamDataSource The data source - * @param newName The new name - * - * @throws EamDbException + * + * @param eamDataSource The data source + * @param newName The new name + * + * @throws EamDbException */ void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException; - + /** * Inserts new Artifact(s) into the database. Should add associated Case and * Data Source first. @@ -275,12 +276,17 @@ public interface EamDb { /** * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * with the eamArtifactType and eamArtifactValues of the given eamArtifact. * - * @param aType EamArtifact.Type to search for - * @param value Value to search for + * @param aType EamArtifact.Type to search for + * @param values The list of correlation values to get + * CorrelationAttributeInstances for * - * @return List of artifact instances for a given type/value + * @return List of artifact instances for a given type with the specified + * values + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException */ List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException; @@ -288,23 +294,35 @@ public interface EamDb { * Retrieves eamArtifact instances from the database that are associated * with the eamArtifactType and eamArtifactValue of the given eamArtifact. * - * @param aType EamArtifact.Type to search for - * @param value Value to search for + * @param aType The type of the artifact + * @param value The correlation value * * @return List of artifact instances for a given type/value + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException */ List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; - + /** * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. + * with the eamArtifactType and eamArtifactValues of the given eamArtifact + * for the specified cases. * - * @param aType EamArtifact.Type to search for - * @param value Value to search for + * @param aType The type of the artifact + * @param values The list of correlation values to get + * CorrelationAttributeInstances for + * @param caseIds The list of central repository case ids to get + * CorrelationAttributeInstances for * - * @return List of artifact instances for a given type/value + * @return List of artifact instances for a given type with the specified + * values for the specified cases + * + * @throws CorrelationAttributeNormalizationException + * @throws EamDbException */ List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException; + /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath @@ -362,7 +380,7 @@ public interface EamDb { * Retrieves number of eamArtifact instances in the database that are * associated with the given data source. * - * @param correlationDataSource Data source to search for + * @param correlationDataSource Data source to search for * * @return Number of artifact instances having caseDisplayName and * dataSource diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index aa02a57839..163d747a4f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -513,15 +513,6 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - */ @Override public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { try { @@ -532,15 +523,6 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - */ @Override public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { try { @@ -551,15 +533,6 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } - /** - * Retrieves eamArtifact instances from the database that are associated - * with the eamArtifactType and eamArtifactValue of the given eamArtifact. - * - * @param aType The type of the artifact - * @param value The correlation value - * - * @return List of artifact instances for a given type/value - */ @Override public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { try { diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java index e9088dbbb6..ee9c70211f 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/AllInterCaseCommonAttributeSearcher.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -64,7 +64,6 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); } Map interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase(), mimeTypesToFilterOn); - return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java index 317c5a26ab..7b927e4e12 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -107,7 +107,6 @@ final public class CommonAttributeCaseSearchResults { * not be more common than * @param resultTypeId the ID of the result type contained in the * metadata - * @param mimeTypesToFilterOn the mimetypes to include in our results * * @return metadata */ @@ -121,7 +120,7 @@ final public class CommonAttributeCaseSearchResults { } Map currentCaseDataSourceMap = metadata.get(currentCaseName); if (currentCaseDataSourceMap == null) { - throw new EamDbException("No data for current case found in results, indicating there are no results and nothing will be filtered"); + return null; } CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance .getDefaultCorrelationTypes() @@ -159,7 +158,6 @@ final public class CommonAttributeCaseSearchResults { * should not be more common than * @param uniqueCaseDataSourceTuples the number of unique data sources in * the CR - * @param mimeTypesToFilterOn the mimetypes to include in our results * * @return a map of correlation value to CommonAttributeValue for results * from the current case @@ -223,7 +221,6 @@ final public class CommonAttributeCaseSearchResults { * should not be more common than * @param uniqueCaseDataSourceTuples the number of unique data sources in * the CR - * @param mimeTypesToInclude the mimetypes to include in our results * * @return true if the value should be filtered and removed from what is * shown to the user, false if the value should not be removed and diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java index 4845218680..38da9bec48 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,18 +22,15 @@ package org.sleuthkit.autopsy.commonpropertiessearch; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; import java.util.logging.Level; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; /** * Stores the results from the various types of common attribute searching @@ -194,7 +191,7 @@ final public class CommonAttributeCountSearchResults { int count = 0; for (CommonAttributeValueList data : this.instanceCountToAttributeValues.values()) { - for (CommonAttributeValue md5 : data.getDelayedMetadataList()) { + for (CommonAttributeValue md5 : data.getDelayedMetadataSet()) { count += md5.getInstanceCount(); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java index 50f29dff13..134d750ace 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java @@ -1,16 +1,16 @@ /* - * + * * Autopsy Forensic Browser - * - * Copyright 2018 Basis Technology Corp. + * + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,10 +36,10 @@ final public class CommonAttributeValueList { * The list of value nodes, which begins empty. */ private final List metadataList; - + /** - * The backing list of value nodes, which will be dynamically loaded - * when requested. + * The backing list of value nodes, which will be dynamically loaded when + * requested. */ private final List delayedMetadataList; @@ -60,44 +60,38 @@ final public class CommonAttributeValueList { } /** - * Get the list of value nodes. Will be empty if - * displayDelayedMetadata() has not been called for the - * parent InstanceCountNode + * Get the list of value nodes. Will be empty if displayDelayedMetadata() + * has not been called for the parent InstanceCountNode + * * @return metadataList the list of nodes */ public List getMetadataList() { return Collections.unmodifiableList(this.metadataList); } - - public Set getMetadataSet() { - return new HashSet<>(this.metadataList); - } - + /** - * Get the delayed list of value nodes. Only use for - * determining how many CommonAttributeValues - * actually exist in the list. - * @return metadataList the list of nodes + * Get the delayed set of value nodes. Only use for determining which values and how many + * CommonAttributeValues actually exist in the list. + * + * @return metadataList the set of nodes */ - List getDelayedMetadataList() { - return Collections.unmodifiableList(this.delayedMetadataList); - } - - Set getDelayedMetadataSet() { + Set getDelayedMetadataSet() { + //Allows nodes to be de-duped return new HashSet<>(this.delayedMetadataList); } - + void removeMetaData(CommonAttributeValue commonVal) { this.delayedMetadataList.remove(commonVal); } - + /** - * Return the size of the backing list, in case - * displayDelayedMetadata() has not be called yet. + * Return the size of the backing list, in case displayDelayedMetadata() has + * not be called yet. + * * @return int the number of matches for this value */ int getCommonAttributeListSize() { - return this.delayedMetadataList.size(); + return this.delayedMetadataList.size(); } /** @@ -113,6 +107,7 @@ final public class CommonAttributeValueList { /** * A a value node to the list, to be loaded later. + * * @param metadata the node to add */ void addMetadataToList(CommonAttributeValue metadata) { diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java index 28c08e3c78..fe6b507f7d 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 9b3500971f..941e95bf6b 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,9 +30,6 @@ import java.util.Set; import java.util.List; import java.util.Map; import java.util.logging.Level; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type; @@ -44,9 +41,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.centralrepository.datamodel.InstanceTableCallback; import org.sleuthkit.autopsy.commonpropertiessearch.AbstractCommonAttributeInstance.NODE_TYPE; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.CaseDbAccessManager; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.HashUtility; @@ -59,21 +53,12 @@ import org.sleuthkit.datamodel.TskCoreException; final class InterCaseSearchResultsProcessor { private static final Logger LOGGER = Logger.getLogger(CommonAttributePanel.class.getName()); + private static final String INTER_CASE_WHERE_CLAUSE = "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; //NON-NLS /** * The CorrelationAttributeInstance.Type this Processor will query on */ private final Type correlationType; - /** - * The initial CorrelationAttributeInstance ids lookup query. - */ - private final String interCaseWhereClause; - - /** - * The single CorrelationAttributeInstance object retrieval query - */ - private final String singleInterCaseWhereClause; - /** * Used in the InterCaseCommonAttributeSearchers to find common attribute * instances and generate nodes at the UI level. @@ -83,16 +68,6 @@ final class InterCaseSearchResultsProcessor { */ InterCaseSearchResultsProcessor(CorrelationAttributeInstance.Type theType) { this.correlationType = theType; - interCaseWhereClause = getInterCaseWhereClause(); - singleInterCaseWhereClause = getSingleInterCaseWhereClause(); - } - - private String getInterCaseWhereClause() { - return "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; - } - - private String getSingleInterCaseWhereClause() { - return "case_id=%s AND (known_status !=%s OR known_status IS NULL)"; } /** @@ -118,6 +93,17 @@ final class InterCaseSearchResultsProcessor { return null; } + /** + * Get the portion of the select query which will get md5 values for files + * from the current case which are potentially being correlated on. + * + * @param mimeTypesToFilterOn the set of mime types to filter on + * + * @return the portion of a query which follows the SELECT keyword for + * finding MD5s which we are correlating on + * + * @throws EamDbException + */ private String getFileQuery(Set mimeTypesToFilterOn) throws EamDbException { String query; query = "md5 as value from tsk_files where known!=" + TskData.FileKnown.KNOWN.getFileKnownValue() + " AND md5 IS NOT NULL"; @@ -132,7 +118,8 @@ final class InterCaseSearchResultsProcessor { * and builds maps of case name to maps of data source name to * CommonAttributeValueList. * - * @param currentCase The current TSK Case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on * * @return map of Case name to Maps of Datasources and their * CommonAttributeValueLists @@ -146,25 +133,26 @@ final class InterCaseSearchResultsProcessor { if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); } else { - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); - } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); - } + } return new HashMap<>(); } /** * Given the current case, fins all intercase common files from the EamDb - * and builds maps of obj id to md5 and case. + * and builds maps of obj id to value and case. * - * @param currentCase The current TSK Case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on + * + * @return map of number of instances to CommonAttributeValueLists */ Map findInterCaseValuesByCount(Case currentCase, Set mimeTypesToFilterOn) { try { @@ -176,27 +164,29 @@ final class InterCaseSearchResultsProcessor { if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); } else { - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); - } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); - } + } return new HashMap<>(); } /** * Given the current case, and a specific case of interest, finds common * files which exist between cases from the EamDb. Builds maps of obj id to - * md5 and case. + * value and case. * - * @param currentCase The current TSK Case. - * @param singleCase The case of interest. Matches must exist in this case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on + * @param singleCase The case of interest. Matches must exist in + * this case. + * + * @return map of number of instances to CommonAttributeValueLists */ Map findSingleInterCaseValuesByCount(Case currentCase, Set mimeTypesToFilterOn, CorrelationCase singleCase) { try { @@ -207,15 +197,13 @@ final class InterCaseSearchResultsProcessor { if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); } else { - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); - } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); } return new HashMap<>(); } @@ -225,13 +213,13 @@ final class InterCaseSearchResultsProcessor { * files which exist between cases from the EamDb. Builds map of case name * to maps of data source name to CommonAttributeValueList. * - * @param currentCase The current TSK Case. + * @param currentCase The current TSK Case. + * @param mimeTypesToFilterOn the set of mime types to filter on + * @param singleCase The case of interest. Matches must exist in + * this case. * * @return map of Case name to Maps of Datasources and their * CommonAttributeValueLists - * - * @param currentCase The current TSK Case. - * @param singleCase The case of interest. Matches must exist in this case. */ Map> findSingleInterCaseValuesByCase(Case currentCase, Set mimeTypesToFilterOn, CorrelationCase singleCase) { try { @@ -243,16 +231,14 @@ final class InterCaseSearchResultsProcessor { if (correlationType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { currentCase.getSleuthkitCase().getCaseDbAccessManager().select(getFileQuery(mimeTypesToFilterOn), instancetableCallback); } else { - dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId, + dbManager.processInstanceTableWhere(correlationType, String.format(INTER_CASE_WHERE_CLAUSE, caseId, TskData.FileKnown.KNOWN.getFileKnownValue()), instancetableCallback); } return instancetableCallback.getInstanceCollatedCommonFiles(); - } catch (EamDbException ex) { + } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); - } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); - } + } return new HashMap<>(); } @@ -319,13 +305,9 @@ final class InterCaseSearchResultsProcessor { } } } - } catch (SQLException ex) { + } catch (SQLException | EamDbException | CorrelationAttributeNormalizationException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS - } catch (EamDbException ex) { - Exceptions.printStackTrace(ex); - } catch (CorrelationAttributeNormalizationException ex) { - Exceptions.printStackTrace(ex); - } + } } Map getInstanceCollatedCommonFiles() { @@ -334,7 +316,7 @@ final class InterCaseSearchResultsProcessor { } /** - * Callback to use with findInterCaseValuesByCount which generates a list of + * Callback to use with findInterCaseValuesByCase which generates a map of maps of * values for common property search */ private class InterCaseByCaseCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback { @@ -401,11 +383,9 @@ final class InterCaseSearchResultsProcessor { } } } - } catch (EamDbException | SQLException ex) { + } catch (EamDbException | SQLException | CorrelationAttributeNormalizationException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS - } catch (CorrelationAttributeNormalizationException ex) { - Exceptions.printStackTrace(ex); - } + } } Map> getInstanceCollatedCommonFiles() { diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java index e098c1ea3d..8d0cca8412 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/SingleInterCaseCommonAttributeSearcher.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -76,7 +76,7 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId); this.correlationCaseName = correlationCase.getDisplayName(); InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType); - Set mimeTypesToFilterOn = new HashSet<>(); + Set mimeTypesToFilterOn = new HashSet<>(); if (isFilterByMedia()) { mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); } From 8c6d035d49cca8d83a53f9ab359f33cf8f3ac83f Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 26 Feb 2019 17:36:42 -0500 Subject: [PATCH 08/11] 4757 auto format of modified files --- .../datamodel/AbstractSqlEamDb.java | 5 ++-- .../datamodel/SqliteEamDb.java | 29 ++++++++++--------- .../CommonAttributeCountSearchResults.java | 2 +- .../CommonAttributeValueList.java | 4 +-- .../InstanceDataSourceNode.java | 3 +- .../InterCaseSearchResultsProcessor.java | 14 ++++----- 6 files changed, 28 insertions(+), 29 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 520908f125..df30f3f5f8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -1058,12 +1058,11 @@ abstract class AbstractSqlEamDb implements EamDb { return getArtifactInstancesByTypeValues(aType, Arrays.asList(value)); } - @Override public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { return getArtifactInstances(prepareGetInstancesSql(aType, values), aType); } - + @Override public List getArtifactInstancesByTypeValuesAndCases(CorrelationAttributeInstance.Type aType, List values, List caseIds) throws EamDbException, CorrelationAttributeNormalizationException { String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); @@ -1077,7 +1076,7 @@ abstract class AbstractSqlEamDb implements EamDb { inValuesBuilder.append("')"); return getArtifactInstances(inValuesBuilder.toString(), aType); } - + /** * Get the select statement for retrieving correlation attribute instances * from the CR for a given type with values matching the specified values diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index 163d747a4f..9e486855bb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -275,7 +275,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { } @Override - public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException{ + public void addDataSourceObjectId(int rowId, long dataSourceObjectId) throws EamDbException { try { acquireExclusiveLock(); super.addDataSourceObjectId(rowId, dataSourceObjectId); @@ -433,14 +433,14 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } - + /** * Changes the name of a data source in the DB - * - * @param eamDataSource The data source - * @param newName The new name - * - * @throws EamDbException + * + * @param eamDataSource The data source + * @param newName The new name + * + * @throws EamDbException */ @Override public void updateDataSourceName(CorrelationDataSource eamDataSource, String newName) throws EamDbException { @@ -451,7 +451,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** * Updates the MD5 hash value in an existing data source in the database. * @@ -466,7 +466,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** * Updates the SHA-1 hash value in an existing data source in the database. * @@ -481,9 +481,10 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + /** - * Updates the SHA-256 hash value in an existing data source in the database. + * Updates the SHA-256 hash value in an existing data source in the + * database. * * @param eamDataSource The data source to update */ @@ -512,7 +513,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseExclusiveLock(); } } - + @Override public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { try { @@ -522,7 +523,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } - + @Override public List getArtifactInstancesByTypeValues(CorrelationAttributeInstance.Type aType, List values) throws EamDbException, CorrelationAttributeNormalizationException { try { @@ -542,7 +543,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } - + /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java index 38da9bec48..c1d2dcb481 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCountSearchResults.java @@ -54,7 +54,7 @@ final public class CommonAttributeCountSearchResults { * common, value of 0 is disabled * @param resultType The type of Correlation Attribute being * searched for - + * */ CommonAttributeCountSearchResults(Map metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType) { //wrap in a new object in case any client code has used an unmodifiable collection diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java index 134d750ace..20ad09c797 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeValueList.java @@ -70,8 +70,8 @@ final public class CommonAttributeValueList { } /** - * Get the delayed set of value nodes. Only use for determining which values and how many - * CommonAttributeValues actually exist in the list. + * Get the delayed set of value nodes. Only use for determining which values + * and how many CommonAttributeValues actually exist in the list. * * @return metadataList the set of nodes */ diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java index fe6b507f7d..8451ec140d 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InstanceDataSourceNode.java @@ -110,8 +110,7 @@ public final class InstanceDataSourceNode extends DisplayableItemNode { } /** - * ChildFactory which builds DisplayableItem from the metadata data - * sources. + * ChildFactory which builds DisplayableItem from the metadata data sources. */ static class FileInstanceNodeFactory extends ChildFactory { diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 941e95bf6b..3f2187aa5b 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -141,7 +141,7 @@ final class InterCaseSearchResultsProcessor { } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); - } + } return new HashMap<>(); } @@ -172,7 +172,7 @@ final class InterCaseSearchResultsProcessor { } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); - } + } return new HashMap<>(); } @@ -238,7 +238,7 @@ final class InterCaseSearchResultsProcessor { return instancetableCallback.getInstanceCollatedCommonFiles(); } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex); - } + } return new HashMap<>(); } @@ -307,7 +307,7 @@ final class InterCaseSearchResultsProcessor { } } catch (SQLException | EamDbException | CorrelationAttributeNormalizationException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS - } + } } Map getInstanceCollatedCommonFiles() { @@ -316,8 +316,8 @@ final class InterCaseSearchResultsProcessor { } /** - * Callback to use with findInterCaseValuesByCase which generates a map of maps of - * values for common property search + * Callback to use with findInterCaseValuesByCase which generates a map of + * maps of values for common property search */ private class InterCaseByCaseCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback, InstanceTableCallback { @@ -385,7 +385,7 @@ final class InterCaseSearchResultsProcessor { } } catch (EamDbException | SQLException | CorrelationAttributeNormalizationException ex) { LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS - } + } } Map> getInstanceCollatedCommonFiles() { From 0b2ccb260881179909bc9508e6545056999248e2 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 26 Feb 2019 17:50:16 -0500 Subject: [PATCH 09/11] 4757 fix pottential NPE when no results are present --- .../CommonAttributeCaseSearchResults.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java index 7b927e4e12..300006daf6 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java @@ -119,8 +119,8 @@ final public class CommonAttributeCaseSearchResults { throw new EamDbException("Unable to get current case while performing filtering", ex); } Map currentCaseDataSourceMap = metadata.get(currentCaseName); - if (currentCaseDataSourceMap == null) { - return null; + if (currentCaseDataSourceMap == null) { //there are no results + return new HashMap<>(); } CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance .getDefaultCorrelationTypes() From e26ce3139bee97b5ef8dbc57671e66faa5f9416c Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 26 Feb 2019 17:55:55 -0500 Subject: [PATCH 10/11] 4757 add non-nls comment to string wither query code --- .../InterCaseSearchResultsProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java index 3f2187aa5b..52f2cec276 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseSearchResultsProcessor.java @@ -106,9 +106,9 @@ final class InterCaseSearchResultsProcessor { */ private String getFileQuery(Set mimeTypesToFilterOn) throws EamDbException { String query; - query = "md5 as value from tsk_files where known!=" + TskData.FileKnown.KNOWN.getFileKnownValue() + " AND md5 IS NOT NULL"; + query = "md5 AS value FROM tsk_files WHERE known!=" + TskData.FileKnown.KNOWN.getFileKnownValue() + " AND md5 IS NOT NULL"; //NON-NLS if (!mimeTypesToFilterOn.isEmpty()) { - query = query + " AND mime_type IS NOT NULL AND mime_type IN ('" + String.join("', '", mimeTypesToFilterOn) + "')"; + query = query + " AND mime_type IS NOT NULL AND mime_type IN ('" + String.join("', '", mimeTypesToFilterOn) + "')"; //NON-NLS } return query; } From e65c135975399f274abcb21389fcfc889442e0dc Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 26 Feb 2019 18:17:43 -0500 Subject: [PATCH 11/11] 4757 fix display of results when everything has been filtered out --- .../CommonAttributeCaseSearchResults.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java index 300006daf6..3ff7d40070 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeCaseSearchResults.java @@ -119,8 +119,9 @@ final public class CommonAttributeCaseSearchResults { throw new EamDbException("Unable to get current case while performing filtering", ex); } Map currentCaseDataSourceMap = metadata.get(currentCaseName); + Map> filteredCaseNameToDataSourcesTree = new HashMap<>(); if (currentCaseDataSourceMap == null) { //there are no results - return new HashMap<>(); + return filteredCaseNameToDataSourcesTree; } CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance .getDefaultCorrelationTypes() @@ -129,13 +130,14 @@ final public class CommonAttributeCaseSearchResults { .findFirst().get(); //Call countUniqueDataSources once to reduce the number of DB queries needed to get the frequencyPercentage Double uniqueCaseDataSourceTuples = EamDb.getInstance().getCountUniqueDataSources().doubleValue(); - Map> filteredCaseNameToDataSourcesTree = new HashMap<>(); Map valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples); for (Entry> mapOfDataSources : Collections.unmodifiableMap(metadata).entrySet()) { if (!mapOfDataSources.getKey().equals(currentCaseName)) { //rebuild the metadata structure with items from the current case substituted for their matches in other cases results we want to filter out removed Map newTreeForCase = createTreeForCase(valuesToKeepCurrentCase, mapOfDataSources.getValue()); - filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase); + if (!newTreeForCase.isEmpty()) { + filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase); + } } } return filteredCaseNameToDataSourcesTree;