From ba0221acf37394e0ffd44667a6ca43761da076be Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 9 Dec 2020 22:15:00 -0500 Subject: [PATCH 1/3] Fixed bottlenecks in UI and caching layer. Updated the date times to reflect the options panel time zone --- .../GeneralPurposeArtifactViewer.java | 19 +++- .../search/DomainSearchArtifactsCache.java | 87 +++++++++++++++++-- .../search/DomainSearchArtifactsLoader.java | 48 +++++----- .../discovery/ui/ArtifactsListPanel.java | 18 ++-- 4 files changed, 132 insertions(+), 40 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/GeneralPurposeArtifactViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/GeneralPurposeArtifactViewer.java index 15ef8c4796..0fe499735d 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/GeneralPurposeArtifactViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/GeneralPurposeArtifactViewer.java @@ -43,9 +43,11 @@ import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ThreadConfined; +import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.discovery.ui.AbstractArtifactDetailsPanel; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.TimeUtilities; import org.sleuthkit.datamodel.TskCoreException; /** @@ -116,7 +118,7 @@ public class GeneralPurposeArtifactViewer extends AbstractArtifactDetailsPanel i } catch (TskCoreException ex) { logger.log(Level.WARNING, "Unable to get attributes for artifact " + artifact.getArtifactID(), ex); } - updateView(artifact.getArtifactTypeID(), attributeMap, dataSourceName, sourceFileName); + updateView(artifact, attributeMap, dataSourceName, sourceFileName); } this.setLayout(this.gridBagLayout); this.revalidate(); @@ -195,7 +197,8 @@ public class GeneralPurposeArtifactViewer extends AbstractArtifactDetailsPanel i * the artifact. */ @ThreadConfined(type = ThreadConfined.ThreadType.AWT) - private void updateView(Integer artifactTypeId, Map> attributeMap, String dataSourceName, String sourceFileName) { + private void updateView(BlackboardArtifact artifact, Map> attributeMap, String dataSourceName, String sourceFileName) { + final Integer artifactTypeId = artifact.getArtifactTypeID(); if (!(artifactTypeId < 1 || artifactTypeId >= Integer.MAX_VALUE)) { addHeader(Bundle.GeneralPurposeArtifactViewer_details_attrHeader()); Integer[] orderingArray = orderingMap.get(artifactTypeId); @@ -206,13 +209,21 @@ public class GeneralPurposeArtifactViewer extends AbstractArtifactDetailsPanel i List attrList = attributeMap.remove(attrId); if (attrList != null) { for (BlackboardAttribute bba : attrList) { - addNameValueRow(bba.getAttributeType().getDisplayName(), bba.getDisplayString()); + if (bba.getAttributeType().getTypeName().startsWith("TSK_DATETIME")) { + addNameValueRow(bba.getAttributeType().getDisplayName(), TimeUtilities.epochToTime(bba.getValueLong(), ContentUtils.getTimeZone(artifact))); + } else { + addNameValueRow(bba.getAttributeType().getDisplayName(), bba.getDisplayString()); + } } } } for (int key : attributeMap.keySet()) { for (BlackboardAttribute bba : attributeMap.get(key)) { - addNameValueRow(bba.getAttributeType().getDisplayName(), bba.getDisplayString()); + if (bba.getAttributeType().getTypeName().startsWith("TSK_DATETIME")) { + addNameValueRow(bba.getAttributeType().getDisplayName(), TimeUtilities.epochToTime(bba.getValueLong(), ContentUtils.getTimeZone(artifact))); + } else { + addNameValueRow(bba.getAttributeType().getDisplayName(), bba.getDisplayString()); + } } } addHeader(Bundle.GeneralPurposeArtifactViewer_details_sourceHeader()); diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java index 200dd12e8c..166065d724 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java @@ -20,21 +20,38 @@ package org.sleuthkit.autopsy.discovery.search; import com.google.common.cache.CacheBuilder; import com.google.common.cache.LoadingCache; +import com.google.common.eventbus.Subscribe; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.sleuthkit.autopsy.discovery.search.DiscoveryEventUtils.SearchStartedEvent; import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; /** * Caches artifact requests. */ public class DomainSearchArtifactsCache { - private static final int MAXIMUM_CACHE_SIZE = 500; - private static final LoadingCache> cache + private static final int MAXIMUM_CACHE_SIZE = 10; + private static final int TIME_TO_LIVE = 5; // In minutes + private static final LoadingCache>> cache = CacheBuilder.newBuilder() .maximumSize(MAXIMUM_CACHE_SIZE) + .expireAfterWrite(TIME_TO_LIVE, TimeUnit.MINUTES) .build(new DomainSearchArtifactsLoader()); - + + // Listen for new search events. When this happens, we should invalidate all the + // entries in the cache. This, along with the 5 minutes expiration, ensures that + // searches get up to date results during ingest. + private static final NewSearchListener newSearchListener = new NewSearchListener(); + static { + DiscoveryEventUtils.getDiscoveryEventBus().register(newSearchListener); + } /** * Get artifact instances that match the requested criteria. If the request * is new, the results will be automatically loaded. @@ -51,12 +68,72 @@ public class DomainSearchArtifactsCache { if (!typeName.startsWith("TSK_WEB")) { throw new IllegalArgumentException("Only web artifacts are valid arguments. Type provided was " + typeName); } - + try { - return cache.get(request); + Map> artifactsByDomain = cache.get(new ArtifactCacheKey(request)); + return artifactsByDomain.getOrDefault(request.getDomain(), Collections.emptyList()); } catch (ExecutionException ex) { //throwing a new exception with the cause so that interrupted exceptions and other causes can be checked inside our wrapper throw new DiscoveryException("Error fetching artifacts from cache for " + request.toString(), ex.getCause()); } } + + /** + * Listener for new searches performed by the user. + */ + static class NewSearchListener { + + @Subscribe + public void listenToSearchStartedEvent(SearchStartedEvent event) { + cache.invalidateAll(); + } + } + + /** + * Key to use for caching. Using only the artifact type and case reference + * will result in greater utilization of the cached artifact instances. + */ + class ArtifactCacheKey { + + private final ARTIFACT_TYPE type; + private final SleuthkitCase caseDatabase; + + private ArtifactCacheKey(DomainSearchArtifactsRequest request) { + this.type = request.getArtifactType(); + this.caseDatabase = request.getSleuthkitCase(); + } + + ARTIFACT_TYPE getType() { + return this.type; + } + + SleuthkitCase getSleuthkitCase() { + return this.caseDatabase; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 67 * hash + Objects.hashCode(this.type); + hash = 67 * hash + Objects.hashCode(this.caseDatabase); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + final ArtifactCacheKey other = (ArtifactCacheKey) obj; + + // The artifact type and case database references must be equal. + return this.type == other.type && + Objects.equals(this.caseDatabase, other.caseDatabase); + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java index 49bdce0c2d..ac774efa11 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java @@ -19,46 +19,46 @@ package org.sleuthkit.autopsy.discovery.search; import com.google.common.cache.CacheLoader; -import java.util.List; import java.util.ArrayList; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; -import org.sleuthkit.datamodel.BlackboardAttribute.Type; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; +import org.sleuthkit.datamodel.BlackboardAttribute; /** - * Loads artifacts for the given request. Searches TSK_DOMAIN and TSK_URL - * attributes for the requested domain name. TSK_DOMAIN is exact match (ignoring - * case). TSK_URL is sub-string match (ignoring case). + * Loads artifacts for the given request. Searches for TSK domain attributes and + * organizes artifacts by those values. */ -public class DomainSearchArtifactsLoader extends CacheLoader> { - - private static final Type TSK_DOMAIN = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DOMAIN); - private static final Type TSK_URL = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_URL); +public class DomainSearchArtifactsLoader extends CacheLoader>> { + private static final BlackboardAttribute.Type TSK_DOMAIN = new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN); + @Override - public List load(DomainSearchArtifactsRequest artifactsRequest) throws TskCoreException, InterruptedException { - final SleuthkitCase caseDb = artifactsRequest.getSleuthkitCase(); - final String normalizedDomain = artifactsRequest.getDomain().toLowerCase(); - final List artifacts = caseDb.getBlackboardArtifacts(artifactsRequest.getArtifactType()); - final List matchingDomainArtifacts = new ArrayList<>(); - + public Map> load(DomainSearchArtifactsCache.ArtifactCacheKey artifactKey) throws TskCoreException, InterruptedException { + final SleuthkitCase caseDb = artifactKey.getSleuthkitCase(); + final ARTIFACT_TYPE type = artifactKey.getType(); + List artifacts = caseDb.getBlackboardArtifacts(type); + + Map> artifactsByDomain = new HashMap<>(); + + // Grab artifacts with matching domain names. for (BlackboardArtifact artifact : artifacts) { if(Thread.currentThread().isInterrupted()) { throw new InterruptedException(); } final BlackboardAttribute tskDomain = artifact.getAttribute(TSK_DOMAIN); - final BlackboardAttribute tskUrl = artifact.getAttribute(TSK_URL); - - if (tskDomain != null && tskDomain.getValueString().equalsIgnoreCase(normalizedDomain)) { - matchingDomainArtifacts.add(artifact); - } else if (tskUrl != null && tskUrl.getValueString().toLowerCase().contains(normalizedDomain)) { - matchingDomainArtifacts.add(artifact); + if (tskDomain != null) { + final String normalizedDomain = tskDomain.getValueString().toLowerCase(); + List artifactsWithDomain = artifactsByDomain.getOrDefault(normalizedDomain, new ArrayList<>()); + artifactsWithDomain.add(artifact); + artifactsByDomain.put(normalizedDomain, artifactsWithDomain); } } - return matchingDomainArtifacts; + return artifactsByDomain; } } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java index 9aa41b3c20..7937ad522d 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java @@ -30,8 +30,10 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ThreadConfined; +import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.TimeUtilities; import org.sleuthkit.datamodel.TskCoreException; /** @@ -242,10 +244,11 @@ class ArtifactsListPanel extends JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { if (columnIndex < 2 || artifactType == BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_CACHE) { + final BlackboardArtifact artifact = getArtifactByRow(rowIndex); try { - for (BlackboardAttribute bba : getArtifactByRow(rowIndex).getAttributes()) { + for (BlackboardAttribute bba : artifact.getAttributes()) { if (!StringUtils.isBlank(bba.getDisplayString())) { - String stringFromAttribute = getStringForColumn(bba, columnIndex); + String stringFromAttribute = getStringForColumn(artifact, bba, columnIndex); if (!StringUtils.isBlank(stringFromAttribute)) { return stringFromAttribute; } @@ -253,7 +256,7 @@ class ArtifactsListPanel extends JPanel { } return getFallbackValue(rowIndex, columnIndex); } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Error getting attributes for artifact " + getArtifactByRow(rowIndex).getArtifactID(), ex); + logger.log(Level.WARNING, "Error getting attributes for artifact " + artifact.getArtifactID(), ex); } } return Bundle.ArtifactsListPanel_value_noValue(); @@ -274,9 +277,9 @@ class ArtifactsListPanel extends JPanel { * the TSK_PATH_ID. */ @ThreadConfined(type = ThreadConfined.ThreadType.AWT) - private String getStringForColumn(BlackboardAttribute bba, int columnIndex) throws TskCoreException { + private String getStringForColumn(BlackboardArtifact artifact, BlackboardAttribute bba, int columnIndex) throws TskCoreException { if (columnIndex == 0 && bba.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID()) { - return bba.getDisplayString(); + return TimeUtilities.epochToTime(bba.getValueLong(), ContentUtils.getTimeZone(artifact)); } else if (columnIndex == 1) { if (artifactType == BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD || artifactType == BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_CACHE) { if (bba.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID.getTypeID()) { @@ -309,9 +312,10 @@ class ArtifactsListPanel extends JPanel { */ @ThreadConfined(type = ThreadConfined.ThreadType.AWT) private String getFallbackValue(int rowIndex, int columnIndex) throws TskCoreException { - for (BlackboardAttribute bba : getArtifactByRow(rowIndex).getAttributes()) { + final BlackboardArtifact artifact = getArtifactByRow(rowIndex); + for (BlackboardAttribute bba : artifact.getAttributes()) { if (columnIndex == 0 && bba.getAttributeType().getTypeName().startsWith("TSK_DATETIME") && !StringUtils.isBlank(bba.getDisplayString())) { - return bba.getDisplayString(); + return TimeUtilities.epochToTime(bba.getValueLong(), ContentUtils.getTimeZone(artifact)); } else if (columnIndex == 1 && bba.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL.getTypeID() && !StringUtils.isBlank(bba.getDisplayString())) { return bba.getDisplayString(); } else if (columnIndex == 1 && bba.getAttributeType().getTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID() && !StringUtils.isBlank(bba.getDisplayString())) { From c4e0dd5832da4c61db9e410c4d8f69e5ec38411c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 10 Dec 2020 08:35:43 -0500 Subject: [PATCH 2/3] Small fix to ensure domain is normalized in both the cache and the loader --- .../autopsy/discovery/search/DomainSearchArtifactsCache.java | 3 ++- .../autopsy/discovery/search/DomainSearchArtifactsLoader.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java index 166065d724..5e43a62c3d 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsCache.java @@ -71,7 +71,8 @@ public class DomainSearchArtifactsCache { try { Map> artifactsByDomain = cache.get(new ArtifactCacheKey(request)); - return artifactsByDomain.getOrDefault(request.getDomain(), Collections.emptyList()); + final String normalizedDomain = request.getDomain().trim().toLowerCase(); + return artifactsByDomain.getOrDefault(normalizedDomain, Collections.emptyList()); } catch (ExecutionException ex) { //throwing a new exception with the cause so that interrupted exceptions and other causes can be checked inside our wrapper throw new DiscoveryException("Error fetching artifacts from cache for " + request.toString(), ex.getCause()); diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java index ac774efa11..196780fa47 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchArtifactsLoader.java @@ -52,7 +52,7 @@ public class DomainSearchArtifactsLoader extends CacheLoader artifactsWithDomain = artifactsByDomain.getOrDefault(normalizedDomain, new ArrayList<>()); artifactsWithDomain.add(artifact); artifactsByDomain.put(normalizedDomain, artifactsWithDomain); From 497fbaf2be397a355b251eb4d0d30557cc85341d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 10 Dec 2020 11:18:19 -0500 Subject: [PATCH 3/3] Renamed visits to page views and added an ability to order within groups by page views --- .../discovery/search/Bundle.properties-MERGED | 9 ++-- .../discovery/search/DiscoveryAttributes.java | 13 ++--- .../discovery/search/DiscoveryKeyUtils.java | 48 ++++++++++--------- .../search/DomainSearchCacheLoader.java | 14 +++--- .../discovery/search/ResultDomain.java | 34 ++++++------- .../discovery/search/ResultsSorter.java | 40 +++++++++++----- .../discovery/ui/Bundle.properties-MERGED | 4 +- .../discovery/ui/DomainSummaryPanel.java | 8 ++-- 8 files changed, 95 insertions(+), 75 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED index 6824493c64..01a2da7b52 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED @@ -7,8 +7,8 @@ DiscoveryAttributes.GroupingAttributeType.interestingItem.displayName=Interestin DiscoveryAttributes.GroupingAttributeType.keywordList.displayName=Keyword DiscoveryAttributes.GroupingAttributeType.mostRecentDate.displayName=Most Recent Activity Date DiscoveryAttributes.GroupingAttributeType.none.displayName=None -DiscoveryAttributes.GroupingAttributeType.numberOfVisits.displayName=Number of Visits DiscoveryAttributes.GroupingAttributeType.object.displayName=Object Detected +DiscoveryAttributes.GroupingAttributeType.pageViews.displayName=Page Views DiscoveryAttributes.GroupingAttributeType.parent.displayName=Parent Folder DiscoveryAttributes.GroupingAttributeType.size.displayName=File Size DiscoveryAttributes.GroupingAttributeType.tag.displayName=Tag @@ -24,10 +24,10 @@ DiscoveryKeyUtils.InterestingItemGroupKey.noSets=None DiscoveryKeyUtils.KeywordListGroupKey.noKeywords=None DiscoveryKeyUtils.MostRecentActivityDateGroupKey.noDate=No Date Available DiscoveryKeyUtils.NoGroupingGroupKey.allFiles=All Files -# {0} - totalVisits -DiscoveryKeyUtils.NumberOfVisitsGroupKey.displayName={0} visits -DiscoveryKeyUtils.NumberOfVisitsGroupKey.noVisits=No visits DiscoveryKeyUtils.ObjectDetectedGroupKey.noSets=None +# {0} - totalVisits +DiscoveryKeyUtils.PageViewsGroupKey.displayName={0} page views +DiscoveryKeyUtils.PageViewsGroupKey.noVisits=No page views # {0} - domain # {1} - artifactType DomainSearchArtifactsRequest.toString.text=Domain: {0} ArtifactType: {1} @@ -52,6 +52,7 @@ FileSorter.SortingMethod.filetype.displayName=File Type FileSorter.SortingMethod.frequency.displayName=Central Repo Frequency FileSorter.SortingMethod.fullPath.displayName=Full Path FileSorter.SortingMethod.keywordlist.displayName=Keyword List Names +FileSorter.SortingMethod.pageViews.displayName=Page Views ResultFile.score.interestingResult.description=At least one instance of the file has an interesting result associated with it. ResultFile.score.notableFile.description=At least one instance of the file was recognized as notable. ResultFile.score.notableTaggedFile.description=At least one instance of the file is tagged with a notable tag. diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java index 0153fe8e65..fe07c360f2 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java @@ -608,13 +608,14 @@ public class DiscoveryAttributes { } /** - * Attribute for grouping/sorting by number of visits. + * Attribute for grouping/sorting domains by number of page views. + * Page views is defined at the number of TSK_WEB_HISTORY artifacts. */ - static class NumberOfVisitsAttribute extends AttributeType { + static class PageViewsAttribute extends AttributeType { @Override public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) { - return new DiscoveryKeyUtils.NumberOfVisitsGroupKey(result); + return new DiscoveryKeyUtils.PageViewsGroupKey(result); } } @@ -742,7 +743,7 @@ public class DiscoveryAttributes { "DiscoveryAttributes.GroupingAttributeType.object.displayName=Object Detected", "DiscoveryAttributes.GroupingAttributeType.mostRecentDate.displayName=Most Recent Activity Date", "DiscoveryAttributes.GroupingAttributeType.firstDate.displayName=First Activity Date", - "DiscoveryAttributes.GroupingAttributeType.numberOfVisits.displayName=Number of Visits", + "DiscoveryAttributes.GroupingAttributeType.pageViews.displayName=Page Views", "DiscoveryAttributes.GroupingAttributeType.none.displayName=None"}) public enum GroupingAttributeType { FILE_SIZE(new FileSizeAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_size_displayName()), @@ -756,7 +757,7 @@ public class DiscoveryAttributes { OBJECT_DETECTED(new ObjectDetectedAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_object_displayName()), MOST_RECENT_DATE(new MostRecentActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_mostRecentDate_displayName()), FIRST_DATE(new FirstActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_firstDate_displayName()), - NUMBER_OF_VISITS(new NumberOfVisitsAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_numberOfVisits_displayName()), + PAGE_VIEWS(new PageViewsAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_pageViews_displayName()), NO_GROUPING(new NoGroupingAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_none_displayName()); private final AttributeType attributeType; @@ -804,7 +805,7 @@ public class DiscoveryAttributes { * @return Enums that can be used to group files. */ public static List getOptionsForGroupingForDomains() { - return Arrays.asList(FREQUENCY, MOST_RECENT_DATE, FIRST_DATE, NUMBER_OF_VISITS); + return Arrays.asList(FREQUENCY, MOST_RECENT_DATE, FIRST_DATE, PAGE_VIEWS); } } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryKeyUtils.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryKeyUtils.java index 9fe4fbf946..f846540986 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryKeyUtils.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryKeyUtils.java @@ -1216,12 +1216,14 @@ public class DiscoveryKeyUtils { } /** - * Key representing the number of visits. + * Key representing the number of page views. + * Page views are defined as the number of TSK_WEB_HISTORY artifacts that match + * a domain value. */ - static class NumberOfVisitsGroupKey extends GroupKey { + static class PageViewsGroupKey extends GroupKey { private final String displayName; - private final Long visits; + private final Long pageViews; /** * Construct a new NumberOfVisitsGroupKey. @@ -1230,19 +1232,19 @@ public class DiscoveryKeyUtils { */ @NbBundle.Messages({ "# {0} - totalVisits", - "DiscoveryKeyUtils.NumberOfVisitsGroupKey.displayName={0} visits", - "DiscoveryKeyUtils.NumberOfVisitsGroupKey.noVisits=No visits"}) - NumberOfVisitsGroupKey(Result result) { + "DiscoveryKeyUtils.PageViewsGroupKey.displayName={0} page views", + "DiscoveryKeyUtils.PageViewsGroupKey.noVisits=No page views"}) + PageViewsGroupKey(Result result) { if (result instanceof ResultDomain) { - Long totalVisits = ((ResultDomain) result).getTotalVisits(); - if (totalVisits == null) { - totalVisits = 0L; + Long totalPageViews = ((ResultDomain) result).getTotalPageViews(); + if (totalPageViews == null) { + totalPageViews = 0L; } - visits = totalVisits; - displayName = Bundle.DiscoveryKeyUtils_NumberOfVisitsGroupKey_displayName(Long.toString(visits)); + pageViews = totalPageViews; + displayName = Bundle.DiscoveryKeyUtils_PageViewsGroupKey_displayName(Long.toString(pageViews)); } else { - displayName = Bundle.DiscoveryKeyUtils_NumberOfVisitsGroupKey_noVisits(); - visits = -1L; + displayName = Bundle.DiscoveryKeyUtils_PageViewsGroupKey_noVisits(); + pageViews = -1L; } } @@ -1257,12 +1259,12 @@ public class DiscoveryKeyUtils { } /** - * Get the number of visits this group is for. + * Get the number of page views this group is for. * - * @return The number of visits this group is for. + * @return The number of page views this group is for. */ - Long getVisits() { - return visits; + Long getPageViews() { + return pageViews; } @Override @@ -1271,19 +1273,19 @@ public class DiscoveryKeyUtils { return true; } - if (!(otherKey instanceof NumberOfVisitsGroupKey)) { + if (!(otherKey instanceof PageViewsGroupKey)) { return false; } - NumberOfVisitsGroupKey visitsKey = (NumberOfVisitsGroupKey) otherKey; - return visits.equals(visitsKey.getVisits()); + PageViewsGroupKey pageViewsKey = (PageViewsGroupKey) otherKey; + return pageViews.equals(pageViewsKey.getPageViews()); } @Override public int compareTo(GroupKey otherGroupKey) { - if (otherGroupKey instanceof NumberOfVisitsGroupKey) { - NumberOfVisitsGroupKey visitsKey = (NumberOfVisitsGroupKey) otherGroupKey; - return Long.compare(getVisits(), visitsKey.getVisits()); + if (otherGroupKey instanceof PageViewsGroupKey) { + PageViewsGroupKey pageViewsKey = (PageViewsGroupKey) otherGroupKey; + return Long.compare(getPageViews(), pageViewsKey.getPageViews()); } else { return compareClassNames(otherGroupKey); } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java index 11b34f5fd4..6e51908dda 100755 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCacheLoader.java @@ -154,12 +154,12 @@ class DomainSearchCacheLoader extends CacheLoader { case BY_DOMAIN_NAME: comparators.add(getDomainNameComparator()); break; + case BY_PAGE_VIEWS: + comparators.add(getPageViewComparator()); default: // The default comparator will be added afterward break; @@ -248,21 +250,31 @@ public class ResultsSorter implements Comparator { return compareStrings(first.getDomain().toLowerCase(), second.getDomain().toLowerCase()); }; } - + /** - * Sorts results by most recent date time. - * - * @return -1 if domain1 comes before domain2, 0 if equal, 1 otherwise. + * Sorts domains by page view count. If a result domain reports it's + * page view count as `null`, then it's assumed to be equivalent to 0. + * + * This comparator sorts results in descending order (largest -> smallest). */ - private static Comparator getMostRecentDateTimeComparator() { - return (Result result1, Result result2) -> { - if (result1.getType() != SearchData.Type.DOMAIN) { + private static Comparator getPageViewComparator() { + return (Result domain1, Result domain2) -> { + if (domain1.getType() != SearchData.Type.DOMAIN) { return 0; } - ResultDomain first = (ResultDomain) result1; - ResultDomain second = (ResultDomain) result2; - return Long.compare(second.getActivityEnd(), first.getActivityEnd()); + ResultDomain first = (ResultDomain) domain1; + ResultDomain second = (ResultDomain) domain2; + + Long firstPageViews = first.getTotalPageViews(); + Long secondPageViews = second.getTotalPageViews(); + if (firstPageViews != null && secondPageViews != null) { + return Long.compare(secondPageViews, firstPageViews); + } else if (firstPageViews == null) { + return 1; + } else { + return -1; + } }; } @@ -318,7 +330,8 @@ public class ResultsSorter implements Comparator { "FileSorter.SortingMethod.frequency.displayName=Central Repo Frequency", "FileSorter.SortingMethod.keywordlist.displayName=Keyword List Names", "FileSorter.SortingMethod.fullPath.displayName=Full Path", - "FileSorter.SortingMethod.domain.displayName=Domain"}) + "FileSorter.SortingMethod.domain.displayName=Domain", + "FileSorter.SortingMethod.pageViews.displayName=Page Views"}) public enum SortingMethod { BY_FILE_NAME(new ArrayList<>(), Bundle.FileSorter_SortingMethod_filename_displayName()), // Sort alphabetically by file name @@ -335,7 +348,8 @@ public class ResultsSorter implements Comparator { BY_FULL_PATH(new ArrayList<>(), Bundle.FileSorter_SortingMethod_fullPath_displayName()), // Sort alphabetically by path BY_DOMAIN_NAME(new ArrayList<>(), - Bundle.FileSorter_SortingMethod_domain_displayName()); + Bundle.FileSorter_SortingMethod_domain_displayName()), + BY_PAGE_VIEWS(new ArrayList<>(), Bundle.FileSorter_SortingMethod_pageViews_displayName()); private final String displayName; private final List requiredAttributes; @@ -381,7 +395,7 @@ public class ResultsSorter implements Comparator { * @return Enum values that can be used to ordering files. */ public static List getOptionsForOrderingDomains() { - return Arrays.asList(BY_DOMAIN_NAME, BY_DATA_SOURCE); + return Arrays.asList(BY_PAGE_VIEWS, BY_DOMAIN_NAME, BY_DATA_SOURCE); } } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED index 7f2aa66a37..a22660cef6 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED @@ -54,8 +54,8 @@ DomainDetailsPanel.miniTimelineTitle.text=Mini Timeline DomainSummaryPanel.activity.text=Activity: {0} to {1} DomainSummaryPanel.downloads.text=Files downloaded: DomainSummaryPanel.loadingImages.text=Loading thumbnail... -DomainSummaryPanel.pages.text=Pages in past 60 days: -DomainSummaryPanel.totalPages.text=Total visits: +DomainSummaryPanel.pages.text=Page views in past 60 days: +DomainSummaryPanel.totalPages.text=Total page views: GroupsListPanel.noDomainResults.message.text=No domains were found for the selected filters.\n\nReminder:\n -The Recent Activity module must be run on each data source you want to find results in.\n -The Central Repository module must be run on each data source if you want to filter or sort by past occurrences.\n -The iOS Analyzer (iLEAPP) module must be run on each data source which contains data from an iOS device.\n GroupsListPanel.noFileResults.message.text=No files were found for the selected filters.\n\nReminder:\n -The File Type Identification module must be run on each data source you want to find results in.\n -The Hash Lookup module must be run on each data source if you want to filter by past occurrence.\n -The Picture Analyzer module must be run on each data source if you are filtering by User Created content. GroupsListPanel.noResults.title.text=No results found diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainSummaryPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainSummaryPanel.java index 74e0d7d0ba..becc7b6511 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainSummaryPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainSummaryPanel.java @@ -142,8 +142,8 @@ class DomainSummaryPanel extends javax.swing.JPanel implements ListCellRenderer< @NbBundle.Messages({"# {0} - startDate", "# {1} - endDate", "DomainSummaryPanel.activity.text=Activity: {0} to {1}", - "DomainSummaryPanel.pages.text=Pages in past 60 days: ", - "DomainSummaryPanel.totalPages.text=Total visits: ", + "DomainSummaryPanel.pages.text=Page views in past 60 days: ", + "DomainSummaryPanel.totalPages.text=Total page views: ", "DomainSummaryPanel.downloads.text=Files downloaded: ", "DomainSummaryPanel.loadingImages.text=Loading thumbnail..."}) @Override @@ -152,8 +152,8 @@ class DomainSummaryPanel extends javax.swing.JPanel implements ListCellRenderer< String startDate = dateFormat.format(new Date(value.getResultDomain().getActivityStart() * 1000)); String endDate = dateFormat.format(new Date(value.getResultDomain().getActivityEnd() * 1000)); activityLabel.setText(Bundle.DomainSummaryPanel_activity_text(startDate, endDate)); - totalVisitsLabel.setText(Bundle.DomainSummaryPanel_totalPages_text() + value.getResultDomain().getTotalVisits()); - pagesLabel.setText(Bundle.DomainSummaryPanel_pages_text() + value.getResultDomain().getVisitsInLast60()); + totalVisitsLabel.setText(Bundle.DomainSummaryPanel_totalPages_text() + value.getResultDomain().getTotalPageViews()); + pagesLabel.setText(Bundle.DomainSummaryPanel_pages_text() + value.getResultDomain().getPageViewsInLast60Days()); filesDownloadedLabel.setText(Bundle.DomainSummaryPanel_downloads_text() + value.getResultDomain().getFilesDownloaded()); if (value.getThumbnail() == null) { numberOfImagesLabel.setText(Bundle.DomainSummaryPanel_loadingImages_text());