From 637e0572b6783fa8f13719acd9b492d1ac2e6380 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 10 Sep 2020 14:19:34 -0400 Subject: [PATCH 1/2] Bug fixes and performance fixes --- .../discovery/search/DiscoveryAttributes.java | 111 +++++++++++++++--- .../discovery/search/DomainSearchCache.java | 13 +- .../search/DomainSearchCacheLoader.java | 21 +++- .../discovery/search/ResultDomain.java | 17 ++- .../discovery/search/ResultsSorter.java | 27 ++++- 5 files changed, 149 insertions(+), 40 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java index 178530e072..ca95a16d08 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java @@ -44,6 +44,8 @@ import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; +import java.util.StringJoiner; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizer; /** * Class which contains the search attributes which can be specified for @@ -220,6 +222,8 @@ public class DiscoveryAttributes { static class FrequencyAttribute extends AttributeType { static final int BATCH_SIZE = 50; // Number of hashes to look up at one time + + static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time @Override public DiscoveryKeyUtils.GroupKey getGroupKey(Result file) { @@ -253,10 +257,13 @@ public class DiscoveryAttributes { List currentFiles = new ArrayList<>(); Set hashesToLookUp = new HashSet<>(); + List domainsToQuery = new ArrayList<>(); + for (Result result : results) { if (result.getKnown() == TskData.FileKnown.KNOWN) { result.setFrequency(SearchData.Frequency.KNOWN); } + if (result.getType() != SearchData.Type.DOMAIN) { ResultFile file = (ResultFile) result; if (file.getFrequency() == SearchData.Frequency.UNKNOWN @@ -265,28 +272,98 @@ public class DiscoveryAttributes { hashesToLookUp.add(file.getFirstInstance().getMd5Hash()); currentFiles.add(file); } + + if (hashesToLookUp.size() >= BATCH_SIZE) { + computeFrequency(hashesToLookUp, currentFiles, centralRepoDb); + + hashesToLookUp.clear(); + currentFiles.clear(); + } } else { - ResultDomain domain = (ResultDomain) result; - try { - CorrelationAttributeInstance.Type domainAttributeType = - centralRepoDb.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID); - Long count = centralRepoDb.getCountArtifactInstancesByTypeValue(domainAttributeType, domain.getDomain()); - domain.setFrequency(SearchData.Frequency.fromCount(count)); - } catch (CentralRepoException ex) { - throw new DiscoveryException("Error encountered querying the central repository.", ex); - } catch (CorrelationAttributeNormalizationException ex) { - logger.log(Level.INFO, "Domain [%s] could not be normalized for central repository querying, skipping...", domain.getDomain()); + ResultDomain domainInstance = (ResultDomain) result; + domainsToQuery.add(domainInstance); + + if (domainsToQuery.size() == DOMAIN_BATCH_SIZE) { + queryDomainFrequency(domainsToQuery, centralRepoDb); + + domainsToQuery.clear(); } } - - if (hashesToLookUp.size() >= BATCH_SIZE) { - computeFrequency(hashesToLookUp, currentFiles, centralRepoDb); + } + + queryDomainFrequency(domainsToQuery, centralRepoDb); + computeFrequency(hashesToLookUp, currentFiles, centralRepoDb); + } + } + + private static void queryDomainFrequency(List domainsToQuery, CentralRepository centralRepository) throws DiscoveryException { + if (domainsToQuery.isEmpty()) { + return; + } + + try { + final Map> resultDomainTable = new HashMap<>(); + final StringJoiner joiner = new StringJoiner(", "); - hashesToLookUp.clear(); - currentFiles.clear(); + final CorrelationAttributeInstance.Type attributeType = centralRepository.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID); + for(ResultDomain domainInstance : domainsToQuery) { + try { + final String domainValue = domainInstance.getDomain(); + final String normalizedDomain = CorrelationAttributeNormalizer.normalize(attributeType, domainValue); + final List bucket = resultDomainTable.getOrDefault(normalizedDomain, new ArrayList<>()); + bucket.add(domainInstance); + resultDomainTable.put(normalizedDomain, bucket); + joiner.add("'" + normalizedDomain + "'"); + } catch (CorrelationAttributeNormalizationException ex) { + logger.log(Level.INFO, String.format("Domain [%s] failed normalization, skipping...", domainInstance.getDomain())); } } - computeFrequency(hashesToLookUp, currentFiles, centralRepoDb); + + final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType); + final String domainFrequencyQuery = " value AS domain_name, COUNT(*) AS frequency " + + "FROM " + tableName + " " + + "WHERE value IN (" + joiner + ") " + + "GROUP BY value"; + + final DomainFrequencyCallback frequencyCallback = new DomainFrequencyCallback(resultDomainTable); + centralRepository.processSelectClause(domainFrequencyQuery, frequencyCallback); + + if (frequencyCallback.getCause() != null) { + throw frequencyCallback.getCause(); + } + } catch (CentralRepoException | SQLException ex) { + throw new DiscoveryException("Fatal exception encountered querying the CR.", ex); + } + } + + private static class DomainFrequencyCallback implements InstanceTableCallback { + + private final Map> domainLookup; + private SQLException sqlCause; + + private DomainFrequencyCallback(Map> domainLookup) { + this.domainLookup = domainLookup; + } + + @Override + public void process(ResultSet resultSet) { + try { + while (resultSet.next()) { + String domain = resultSet.getString("domain_name"); + Long frequency = resultSet.getLong("frequency"); + + List domainInstances = domainLookup.get(domain); + for(ResultDomain domainInstance : domainInstances) { + domainInstance.setFrequency(SearchData.Frequency.fromCount(frequency)); + } + } + } catch (SQLException ex) { + this.sqlCause = ex; + } + } + + SQLException getCause() { + return this.sqlCause; } } @@ -644,7 +721,7 @@ public class DiscoveryAttributes { return Arrays.asList(FILE_SIZE, FREQUENCY, PARENT_PATH, OBJECT_DETECTED, HASH_LIST_NAME, INTERESTING_ITEM_SET); } } - + /** * Computes the CR frequency of all the given hashes and updates the list of * files. diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCache.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCache.java index fb29e41078..fca63177e5 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCache.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCache.java @@ -36,17 +36,10 @@ import org.sleuthkit.datamodel.SleuthkitCase; class DomainSearchCache { private static final int MAXIMUM_CACHE_SIZE = 10; - private final LoadingCache>> cache; - - DomainSearchCache() { - this(new DomainSearchCacheLoader()); - } - - DomainSearchCache(DomainSearchCacheLoader cacheLoader) { - this.cache = CacheBuilder.newBuilder() + private static final LoadingCache>> cache = + CacheBuilder.newBuilder() .maximumSize(MAXIMUM_CACHE_SIZE) - .build(cacheLoader); - } + .build(new DomainSearchCacheLoader()); /** * Get domain search results matching the given parameters. If no results diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java index 59c47429c4..9bc6e1f417 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java @@ -95,7 +95,7 @@ class DomainSearchCacheLoader extends CacheLoader(); this.skc = skc; } @@ -235,6 +239,8 @@ class DomainSearchCacheLoader extends CacheLoader { case BY_DOMAIN_NAME: comparators.add(getDomainNameComparator()); break; + case BY_MOST_RECENT_DATE_TIME: + comparators.add(getMostRecentDateTimeComparator()); + break; default: // The default comparator will be added afterward break; @@ -112,7 +115,7 @@ public class ResultsSorter implements Comparator { private static Comparator getTypeComparator() { return (Result result1, Result result2) -> Integer.compare(result1.getType().getRanking(), result2.getType().getRanking()); } - + /** * Compare files using a concatenated version of keyword list names. * Alphabetical by the list names with files with no keyword list hits going @@ -244,6 +247,21 @@ public class ResultsSorter implements Comparator { return compareStrings(first.getDomain().toLowerCase(), second.getDomain().toLowerCase()); }; } + + /** + * Sorts results by most recent date time + */ + private static Comparator getMostRecentDateTimeComparator() { + return (Result result1, Result result2) -> { + if(result1.getType() != SearchData.Type.DOMAIN) { + return 0; + } + + ResultDomain first = (ResultDomain) result1; + ResultDomain second = (ResultDomain) result2; + return Long.compare(second.getActivityEnd(), first.getActivityEnd()); + }; + } /** * A final default comparison between two ResultFile objects. Currently this @@ -297,7 +315,8 @@ public class ResultsSorter implements Comparator { "FileSorter.SortingMethod.frequency.displayName=Central Repo Frequency", "FileSorter.SortingMethod.keywordlist.displayName=Keyword List Names", "FileSorter.SortingMethod.fullPath.displayName=Full Path", - "FileSorter.SortingMethod.domain.displayName=Domain"}) + "FileSorter.SortingMethod.domain.displayName=Domain", + "FileSorter.SortingMethod.dateTime.displayName=Most Recent Date"}) public enum SortingMethod { BY_FILE_NAME(new ArrayList<>(), Bundle.FileSorter_SortingMethod_filename_displayName()), // Sort alphabetically by file name @@ -314,7 +333,9 @@ public class ResultsSorter implements Comparator { BY_FULL_PATH(new ArrayList<>(), Bundle.FileSorter_SortingMethod_fullPath_displayName()), // Sort alphabetically by path BY_DOMAIN_NAME(new ArrayList<>(), - Bundle.FileSorter_SortingMethod_domain_displayName()); + Bundle.FileSorter_SortingMethod_domain_displayName()), + BY_MOST_RECENT_DATE_TIME(new ArrayList<>(), + Bundle.FileSorter_SortingMethod_dateTime_displayName()); private final String displayName; private final List requiredAttributes; From e36e32fd6d20dbc6f3cb655b20be8124e1a05c28 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 11 Sep 2020 10:02:25 -0400 Subject: [PATCH 2/2] Added tests, removed some logic added for data source domains table --- Core/ivy.xml | 3 + Core/nbproject/project.properties | 4 + Core/nbproject/project.xml | 32 +- .../discovery/search/Bundle.properties-MERGED | 1 + .../search/DomainSearchCacheLoader.java | 11 +- .../discovery/search/ResultDomain.java | 11 +- .../discovery/search/ResultsSorter.java | 10 +- .../search/DomainSearchCacheLoaderTest.java | 157 +++++++ .../discovery/search/DomainSearchTest.java | 413 ++++++++++++++++++ .../search/DomainSearchTestUtils.java | 51 +++ 10 files changed, 658 insertions(+), 35 deletions(-) create mode 100755 Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoaderTest.java create mode 100755 Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTest.java create mode 100755 Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java diff --git a/Core/ivy.xml b/Core/ivy.xml index 63fdd9ed92..1b2acba60d 100644 --- a/Core/ivy.xml +++ b/Core/ivy.xml @@ -54,6 +54,9 @@ + + + diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 8dbd7f8d92..37d8de5814 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -13,6 +13,8 @@ file.reference.bcpkix-jdk15on-1.60.jar=release\\modules\\ext\\bcpkix-jdk15on-1.6 file.reference.bcprov-ext-jdk15on-1.54.jar=release/modules/ext/bcprov-ext-jdk15on-1.54.jar file.reference.bcprov-jdk15on-1.60.jar=release\\modules\\ext\\bcprov-jdk15on-1.60.jar file.reference.boilerpipe-1.1.0.jar=release\\modules\\ext\\boilerpipe-1.1.0.jar +file.reference.byte-buddy-1.10.13.jar=release\\modules\\ext\\byte-buddy-1.10.13.jar +file.reference.byte-buddy-agent-1.10.13.jar=release\\modules\\ext\\byte-buddy-agent-1.10.13.jar file.reference.c3p0-0.9.5.jar=release/modules/ext/c3p0-0.9.5.jar file.reference.cdm-4.5.5.jar=release\\modules\\ext\\cdm-4.5.5.jar file.reference.commons-codec-1.11.jar=release\\modules\\ext\\commons-codec-1.11.jar @@ -68,7 +70,9 @@ file.reference.jython-standalone-2.7.0.jar=release/modules/ext/jython-standalone file.reference.libphonenumber-3.5.jar=release/modules/ext/libphonenumber-3.5.jar file.reference.mchange-commons-java-0.2.9.jar=release/modules/ext/mchange-commons-java-0.2.9.jar file.reference.metadata-extractor-2.11.0.jar=release\\modules\\ext\\metadata-extractor-2.11.0.jar +file.reference.mockito-core-3.5.7.jar=release\\modules\\ext\\mockito-core-3.5.7.jar file.reference.netcdf4-4.5.5.jar=release\\modules\\ext\\netcdf4-4.5.5.jar +file.reference.objenesis-3.1.jar=release\\modules\\ext\\objenesis-3.1.jar file.reference.openjson-1.0.10.jar=release\\modules\\ext\\openjson-1.0.10.jar file.reference.opennlp-tools-1.9.1.jar=release\\modules\\ext\\opennlp-tools-1.9.1.jar file.reference.parso-2.0.10.jar=release\\modules\\ext\\parso-2.0.10.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 61e6a86b04..5abd3666b7 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -423,6 +423,10 @@ ext/okhttp-2.7.5.jar release/modules/ext/okhttp-2.7.5.jar + + ext/byte-buddy-1.10.13.jar + release\modules\ext\byte-buddy-1.10.13.jar + ext/libphonenumber-3.5.jar release/modules/ext/libphonenumber-3.5.jar @@ -471,10 +475,6 @@ ext/commons-pool2-2.4.2.jar release/modules/ext/commons-pool2-2.4.2.jar - - ext/sleuthkit-4.10.0.jar - release/modules/ext/sleuthkit-4.10.0.jar - ext/jxmapviewer2-2.4.jar release/modules/ext/jxmapviewer2-2.4.jar @@ -611,6 +611,14 @@ ext/grib-4.5.5.jar release\modules\ext\grib-4.5.5.jar + + ext/sleuthkit-4.10.0.jar + release/modules/ext/sleuthkit-4.10.0.jar + + + ext/sleuthkit-caseuco-4.10.0.jar + release/modules/ext/sleuthkit-caseuco-4.10.0.jar + ext/gax-1.44.0.jar release/modules/ext/gax-1.44.0.jar @@ -647,6 +655,10 @@ ext/decodetect-core-0.3.jar release/modules/ext/decodetect-core-0.3.jar + + ext/mockito-core-3.5.7.jar + release\modules\ext\mockito-core-3.5.7.jar + ext/jbig2-imageio-3.0.2.jar release\modules\ext\jbig2-imageio-3.0.2.jar @@ -667,6 +679,10 @@ ext/curator-recipes-2.8.0.jar release/modules/ext/curator-recipes-2.8.0.jar + + ext/objenesis-3.1.jar + release\modules\ext\objenesis-3.1.jar + ext/tagsoup-1.2.1.jar release\modules\ext\tagsoup-1.2.1.jar @@ -779,10 +795,6 @@ ext/curator-client-2.8.0.jar release/modules/ext/curator-client-2.8.0.jar - - ext/sleuthkit-caseuco-4.10.0.jar - release/modules/ext/sleuthkit-caseuco-4.10.0.jar - ext/fontbox-2.0.13.jar release\modules\ext\fontbox-2.0.13.jar @@ -831,6 +843,10 @@ ext/jutf7-1.0.0.jar release/modules/ext/jutf7-1.0.0.jar + + ext/byte-buddy-agent-1.10.13.jar + release\modules\ext\byte-buddy-agent-1.10.13.jar + ext/batik-awt-util-1.6.jar release/modules/ext/batik-awt-util-1.6.jar diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED index fbb3240018..b31ec5db26 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED @@ -34,6 +34,7 @@ FileSearchFiltering.TagsFilter.desc=Tagged {0} FileSearchFiltering.TagsFilter.or=, FileSearchFiltering.UserCreatedFilter.desc=that contain EXIF data FileSorter.SortingMethod.datasource.displayName=Data Source +FileSorter.SortingMethod.domain.displayName=Domain FileSorter.SortingMethod.filename.displayName=File Name FileSorter.SortingMethod.filesize.displayName=File Size FileSorter.SortingMethod.filetype.displayName=File Type diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java index 9bc6e1f417..999aab0425 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java @@ -136,10 +136,6 @@ class DomainSearchCacheLoader extends CacheLoader { case BY_DOMAIN_NAME: comparators.add(getDomainNameComparator()); break; - case BY_MOST_RECENT_DATE_TIME: - comparators.add(getMostRecentDateTimeComparator()); - break; default: // The default comparator will be added afterward break; @@ -315,8 +312,7 @@ public class ResultsSorter implements Comparator { "FileSorter.SortingMethod.frequency.displayName=Central Repo Frequency", "FileSorter.SortingMethod.keywordlist.displayName=Keyword List Names", "FileSorter.SortingMethod.fullPath.displayName=Full Path", - "FileSorter.SortingMethod.domain.displayName=Domain", - "FileSorter.SortingMethod.dateTime.displayName=Most Recent Date"}) + "FileSorter.SortingMethod.domain.displayName=Domain"}) public enum SortingMethod { BY_FILE_NAME(new ArrayList<>(), Bundle.FileSorter_SortingMethod_filename_displayName()), // Sort alphabetically by file name @@ -333,9 +329,7 @@ public class ResultsSorter implements Comparator { BY_FULL_PATH(new ArrayList<>(), Bundle.FileSorter_SortingMethod_fullPath_displayName()), // Sort alphabetically by path BY_DOMAIN_NAME(new ArrayList<>(), - Bundle.FileSorter_SortingMethod_domain_displayName()), - BY_MOST_RECENT_DATE_TIME(new ArrayList<>(), - Bundle.FileSorter_SortingMethod_dateTime_displayName()); + Bundle.FileSorter_SortingMethod_domain_displayName()); private final String displayName; private final List requiredAttributes; diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoaderTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoaderTest.java new file mode 100755 index 0000000000..2a75ca2dc7 --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoaderTest.java @@ -0,0 +1,157 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.discovery.search; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import static org.mockito.Mockito.*; +import static org.junit.Assert.*; +import org.junit.Test; +import org.sleuthkit.autopsy.discovery.search.DiscoveryKeyUtils.GroupKey; +import org.sleuthkit.autopsy.discovery.search.DiscoveryKeyUtils.SearchKey; +import org.sleuthkit.datamodel.TskCoreException; + +public class DomainSearchCacheLoaderTest { + + @Test + public void load_GroupByDataSourceSortByGroupNameAndDomain() throws DiscoveryException, TskCoreException, SQLException { + DomainSearchCacheLoader loader = mock(DomainSearchCacheLoader.class); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com", 10, 100, 5, 4, 110), + DomainSearchTestUtils.mockDomainResult("yahoo.com", 1, 5, 7, 20, 100), + DomainSearchTestUtils.mockDomainResult("google.com", 5, 20, 1, 4, 105), + DomainSearchTestUtils.mockDomainResult("facebook.com", 2, 2, 1, 3, 110), + DomainSearchTestUtils.mockDomainResult("abc.com", 1, 2, 3, 4, 100), + DomainSearchTestUtils.mockDomainResult("xyz.com", 1, 2, 3, 4, 20) + ); + + SearchKey key = new SearchKey(null, new ArrayList<>(), + new DiscoveryAttributes.DataSourceAttribute(), + Group.GroupSortingAlgorithm.BY_GROUP_NAME, + ResultsSorter.SortingMethod.BY_DOMAIN_NAME); + + when(loader.getResultDomainsFromDatabase(key)).thenReturn(domains); + when(loader.load(key)).thenCallRealMethod(); + Map> results = loader.load(key); + assertEquals(4, results.size()); + for(List group : results.values()) { + ResultDomain previous = null; + for(Result result : group) { + ResultDomain current = (ResultDomain) result; + if (previous != null) { + assertTrue(previous.getDomain().compareTo(current.getDomain()) < 0); + } + previous = current; + } + } + } + + @Test + public void load_GroupByNothingByGroupNameAndDomain() throws DiscoveryException, TskCoreException, SQLException { + DomainSearchCacheLoader loader = mock(DomainSearchCacheLoader.class); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com", 10, 100, 5, 4, 110), + DomainSearchTestUtils.mockDomainResult("yahoo.com", 1, 5, 7, 20, 100), + DomainSearchTestUtils.mockDomainResult("facebook.com", 2, 2, 1, 3, 110), + DomainSearchTestUtils.mockDomainResult("abc.com", 1, 2, 3, 4, 100), + DomainSearchTestUtils.mockDomainResult("xyz.com", 1, 2, 3, 4, 20) + ); + + SearchKey key = new SearchKey(null, new ArrayList<>(), + new DiscoveryAttributes.NoGroupingAttribute(), + Group.GroupSortingAlgorithm.BY_GROUP_NAME, + ResultsSorter.SortingMethod.BY_DOMAIN_NAME); + + when(loader.getResultDomainsFromDatabase(key)).thenReturn(domains); + when(loader.load(key)).thenCallRealMethod(); + Map> results = loader.load(key); + assertEquals(1, results.size()); + for(List group : results.values()) { + ResultDomain previous = null; + for(Result result : group) { + ResultDomain current = (ResultDomain) result; + if (previous != null) { + assertTrue(previous.getDomain().compareTo(current.getDomain()) < 0); + } + previous = current; + } + } + } + + @Test + public void load_GroupByNothingSortByNameAndDataSource() throws DiscoveryException, TskCoreException, SQLException { + DomainSearchCacheLoader loader = mock(DomainSearchCacheLoader.class); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com", 10, 100, 5, 4, 110), + DomainSearchTestUtils.mockDomainResult("yahoo.com", 1, 5, 7, 20, 100) + ); + + SearchKey key = new SearchKey(null, new ArrayList<>(), + new DiscoveryAttributes.NoGroupingAttribute(), + Group.GroupSortingAlgorithm.BY_GROUP_NAME, + ResultsSorter.SortingMethod.BY_DATA_SOURCE); + + when(loader.getResultDomainsFromDatabase(key)).thenReturn(domains); + when(loader.load(key)).thenCallRealMethod(); + Map> results = loader.load(key); + assertEquals(1, results.size()); + for(List group : results.values()) { + ResultDomain previous = null; + for(Result result : group) { + ResultDomain current = (ResultDomain) result; + if (previous != null) { + assertTrue(Long.compare(previous.getDataSource().getId(), current.getDataSource().getId()) < 0); + } + previous = current; + } + } + } + + @Test + public void load_GroupByDataSourceBySizeAndName() throws DiscoveryException, TskCoreException, SQLException { + DomainSearchCacheLoader loader = mock(DomainSearchCacheLoader.class); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com", 10, 100, 5, 4, 110), + DomainSearchTestUtils.mockDomainResult("yahoo.com", 1, 5, 7, 20, 100) + ); + + SearchKey key = new SearchKey(null, new ArrayList<>(), + new DiscoveryAttributes.DataSourceAttribute(), + Group.GroupSortingAlgorithm.BY_GROUP_SIZE, + ResultsSorter.SortingMethod.BY_DOMAIN_NAME); + + when(loader.getResultDomainsFromDatabase(key)).thenReturn(domains); + when(loader.load(key)).thenCallRealMethod(); + Map> results = loader.load(key); + assertEquals(2, results.size()); + for(List group : results.values()) { + ResultDomain previous = null; + for(Result result : group) { + ResultDomain current = (ResultDomain) result; + if (previous != null) { + assertTrue(previous.getDomain().compareTo(current.getDomain()) < 0); + } + previous = current; + } + } + } +} diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTest.java new file mode 100755 index 0000000000..9ddd7db0bf --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTest.java @@ -0,0 +1,413 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.discovery.search; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; + +import static org.mockito.Mockito.*; +import static org.junit.Assert.*; +import org.sleuthkit.autopsy.discovery.search.DiscoveryKeyUtils.GroupKey; + +public class DomainSearchTest { + + @Test + public void groupSizes_SingleGroup_ShouldHaveSizeFour() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + Map> dummyData = new HashMap>() { + { + put(groupOne, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")) + ); + } + }; + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + Map sizes = domainSearch.getGroupSizes(null, + new ArrayList<>(), null, null, null, null, null); + assertEquals(4, sizes.get(groupOne).longValue()); + } + + @Test + public void groupSizes_MultipleGroups_ShouldHaveCorrectGroupSizes() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + DummyKey groupTwo = new DummyKey("2"); + DummyKey groupThree = new DummyKey("3"); + + Map> dummyData = new HashMap>() { + { + put(groupOne, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")) + ); + put(groupTwo, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("facebook.com"), + DomainSearchTestUtils.mockDomainResult("spotify.com"), + DomainSearchTestUtils.mockDomainResult("netbeans.com")) + ); + put(groupThree, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("youtube.com")) + ); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + Map sizes = domainSearch.getGroupSizes(null, + new ArrayList<>(), null, null, null, null, null); + assertEquals(4, sizes.get(groupOne).longValue()); + assertEquals(3, sizes.get(groupTwo).longValue()); + assertEquals(1, sizes.get(groupThree).longValue()); + } + + @Test + public void groupSizes_EmptyGroup_ShouldBeSizeZero() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(new HashMap<>()); + + DomainSearch domainSearch = new DomainSearch(cache); + Map sizes = domainSearch.getGroupSizes(null, + new ArrayList<>(), null, null, null, null, null); + assertEquals(0, sizes.size()); + } + + @Test + public void getDomains_SingleGroupFullPage_ShouldContainAllDomains() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")); + + Map> dummyData = new HashMap>() { + { + put(groupOne, domains); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, 0, 3, null, null); + assertEquals(4, firstPage.size()); + for (int i = 0; i < firstPage.size(); i++) { + assertEquals(domains.get(i), firstPage.get(i)); + } + } + + @Test + public void getDomains_SingleGroupOverSizedPage_ShouldContainAllDomains() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")); + + Map> dummyData = new HashMap>() { + { + put(groupOne, domains); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, 0, 100, null, null); + assertEquals(4, firstPage.size()); + for (int i = 0; i < firstPage.size(); i++) { + assertEquals(domains.get(i), firstPage.get(i)); + } + } + + @Test + public void getDomains_SingleGroupHalfPage_ShouldContainHalfDomains() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")); + + Map> dummyData = new HashMap>() { + { + put(groupOne, domains); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, 0, 2, null, null); + assertEquals(2, firstPage.size()); + for (int i = 0; i < firstPage.size(); i++) { + assertEquals(domains.get(i), firstPage.get(i)); + } + } + + @Test + public void getDomains_SingleGroupLastPageLastDomain_ShouldContainLastDomain() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")); + + Map> dummyData = new HashMap>() { + { + put(groupOne, domains); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, 3, 1, null, null); + assertEquals(1, firstPage.size()); + assertEquals(domains.get(domains.size() - 1), firstPage.get(0)); + } + + @Test + public void getDomains_SingleGroupOversizedOffset_ShouldContainNoDomains() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")); + + Map> dummyData = new HashMap>() { + { + put(groupOne, domains); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, 20, 5, null, null); + assertEquals(4, firstPage.size()); + for (int i = 0; i < firstPage.size(); i++) { + assertEquals(domains.get(i), firstPage.get(i)); + } + } + + @Test + public void getDomains_SingleGroupZeroSizedPage_ShouldContainNoDomains() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")); + + Map> dummyData = new HashMap>() { + { + put(groupOne, domains); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, 0, 0, null, null); + assertEquals(0, firstPage.size()); + } + + @Test + public void getDomains_MultipleGroupsFullPage_ShouldContainAllDomainsInGroup() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + DummyKey groupTwo = new DummyKey("2"); + DummyKey groupThree = new DummyKey("3"); + + Map> dummyData = new HashMap>() { + { + put(groupOne, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")) + ); + put(groupTwo, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("facebook.com"), + DomainSearchTestUtils.mockDomainResult("spotify.com"), + DomainSearchTestUtils.mockDomainResult("netbeans.com")) + ); + put(groupThree, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("youtube.com")) + ); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, 0, 3, null, null); + assertEquals(4, firstPage.size()); + } + + @Test + public void getDomains_MultipleGroupsHalfPage_ShouldContainHalfDomainsInGroup() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + DummyKey groupTwo = new DummyKey("2"); + DummyKey groupThree = new DummyKey("3"); + + Map> dummyData = new HashMap>() { + { + put(groupOne, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com")) + ); + put(groupTwo, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("facebook.com"), + DomainSearchTestUtils.mockDomainResult("spotify.com"), + DomainSearchTestUtils.mockDomainResult("netbeans.com")) + ); + put(groupThree, Arrays.asList( + DomainSearchTestUtils.mockDomainResult("youtube.com")) + ); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + List firstPage = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupTwo, 1, 2, null, null); + assertEquals(2, firstPage.size()); + for (int i = 0; i < firstPage.size(); i++) { + assertEquals(dummyData.get(groupTwo).get(i + 1), firstPage.get(i)); + } + } + + @Test + public void getDomains_SingleGroupSimulatedPaging_ShouldPageThroughAllDomains() throws DiscoveryException { + DomainSearchCache cache = mock(DomainSearchCache.class); + + DummyKey groupOne = new DummyKey("1"); + List domains = Arrays.asList( + DomainSearchTestUtils.mockDomainResult("google.com"), + DomainSearchTestUtils.mockDomainResult("yahoo.com"), + DomainSearchTestUtils.mockDomainResult("bing.com"), + DomainSearchTestUtils.mockDomainResult("amazon.com"), + DomainSearchTestUtils.mockDomainResult("facebook.com"), + DomainSearchTestUtils.mockDomainResult("capitalone.com"), + DomainSearchTestUtils.mockDomainResult("spotify.com"), + DomainSearchTestUtils.mockDomainResult("netsuite.com")); + + Map> dummyData = new HashMap>() { + { + put(groupOne, domains); + } + }; + + when(cache.get(null, new ArrayList<>(), null, null, null, null, null)).thenReturn(dummyData); + + DomainSearch domainSearch = new DomainSearch(cache); + + int start = 0; + int size = 2; + while (start + size <= domains.size()) { + List page = domainSearch.getDomainsInGroup(null, + new ArrayList<>(), null, null, null, groupOne, start, size, null, null); + assertEquals(2, page.size()); + for(int i = 0; i < page.size(); i++) { + assertEquals(domains.get(start + i), page.get(i)); + } + + start += size; + } + } + + private class DummyKey extends GroupKey { + + private final String name; + + public DummyKey(String name) { + this.name = name; + } + + @Override + String getDisplayName() { + return name; + } + + @Override + public boolean equals(Object otherKey) { + if (otherKey instanceof GroupKey) { + return this.getDisplayName().equals(((GroupKey) otherKey).getDisplayName()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return this.name.hashCode(); + } + + @Override + public int compareTo(GroupKey o) { + return this.getDisplayName().compareTo(o.getDisplayName()); + } + } +} diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java new file mode 100755 index 0000000000..a0f19c17a4 --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java @@ -0,0 +1,51 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.discovery.search; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import org.sleuthkit.datamodel.Content; + +/** + * Mock utility methods for DomainSearchTests + */ +public class DomainSearchTestUtils { + + private DomainSearchTestUtils() { + + } + + public static ResultDomain mockDomainResult(String domain, long start, long end, + long visits, long filesDownloaded, long dataSourceId) { + Content dataSource = mockDataSource(dataSourceId); + return new ResultDomain(domain, start, end, + visits, filesDownloaded, dataSource); + } + + public static ResultDomain mockDomainResult(String domain) { + return DomainSearchTestUtils.mockDomainResult(domain, 0, 0, 0, 0, 0); + } + + public static Content mockDataSource(long dataSourceId) { + Content dataSource = mock(Content.class); + when(dataSource.getName()).thenReturn(""); + when(dataSource.getId()).thenReturn(dataSourceId); + return dataSource; + } +}