Added web categories to domains for grouping

Core/src/org/sleuthkit/autopsy/discovery/search/Bundle.properties-MERGED

@@ -13,6 +13,7 @@ DiscoveryAttributes.GroupingAttributeType.parent.displayName=Parent Folder
 DiscoveryAttributes.GroupingAttributeType.previouslyNotable.displayName=Previous Notability
 DiscoveryAttributes.GroupingAttributeType.size.displayName=File Size
 DiscoveryAttributes.GroupingAttributeType.tag.displayName=Tag
+DiscoveryAttributes.GroupingAttributeType.webCategory.displayName=Domain Category
 # {0} - Data source name
 # {1} - Data source ID
 DiscoveryKeyUtils.DataSourceGroupKey.datasourceAndID={0}(ID: {1})
@@ -50,6 +51,7 @@ FileSorter.SortingMethod.filetype.displayName=File Type
 FileSorter.SortingMethod.frequency.displayName=Central Repo Frequency
 FileSorter.SortingMethod.fullPath.displayName=Full Path
 FileSorter.SortingMethod.keywordlist.displayName=Keyword List Names
+ResultDomain_getDefaultCategory=Uncategorized
 ResultFile.score.interestingResult.description=At least one instance of the file has an interesting result associated with it.
 ResultFile.score.notableFile.description=At least one instance of the file was recognized as notable.
 ResultFile.score.notableTaggedFile.description=At least one instance of the file is tagged with a notable tag.

Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java

@@ -46,6 +46,7 @@ import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 import java.util.StringJoiner;
 import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizer;
+import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_CATEGORIZATION;
 
 /**
  * Class which contains the search attributes which can be specified for
@@ -140,6 +141,59 @@ public class DiscoveryAttributes {
         }
     }
     
+    /**
+     * Attribute for grouping/sorting by domain category (TSK_WEB_CATEGORY artifacts).
+     */
+    static class DomainCategoryAttribute extends AttributeType {
+
+        @Override
+        public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) {
+            return new DiscoveryKeyUtils.DomainCategoryGroupKey(result);
+        }
+        
+        @Override
+        public void addAttributeToResults(List<Result> results, SleuthkitCase caseDb,
+                CentralRepository centralRepoDb) throws DiscoveryException {
+            try {
+                Map<String, String> domainsToCategories = getDomainsWithWebCategories(caseDb);
+                for (Result result : results) {
+                    if (result instanceof ResultDomain) {
+                        ResultDomain domain = (ResultDomain) result;
+                        String webCategory = domainsToCategories.get(domain.getDomain());
+                        domain.setWebCategory(webCategory);
+                    }
+                }
+            } catch (TskCoreException | InterruptedException ex) {
+                throw new DiscoveryException("Error fetching TSK_WEB_CATEGORY artifacts from the database", ex);
+            }
+        }
+        
+        /**
+         * Loads all TSK_WEB_CATEGORY artifacts and maps the domain attribute to the category name attribute.
+         * Each ResultDomain is then parsed and matched against this map of values.
+         */
+        private Map<String, String> getDomainsWithWebCategories(SleuthkitCase caseDb) throws TskCoreException, InterruptedException {
+            Map<String, String> domainToCategory = new HashMap<>();
+
+            for (BlackboardArtifact artifact : caseDb.getBlackboardArtifacts(TSK_WEB_CATEGORIZATION)) {
+                if (Thread.currentThread().isInterrupted()) {
+                    throw new InterruptedException();
+                }
+
+                BlackboardAttribute webCategory = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME));
+                BlackboardAttribute domain = artifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN));
+
+                if (webCategory != null && domain != null) {
+                    String webCatDisplayName = webCategory.getValueString();
+                    String domainDisplayName = domain.getValueString().trim().toLowerCase();
+                    domainToCategory.put(domainDisplayName, webCatDisplayName);
+                }
+            }
+
+           return domainToCategory;
+        }
+    }
+
     /**
      * Attribute for grouping/sorting by keyword lists.
      */
@@ -866,7 +920,8 @@ public class DiscoveryAttributes {
         "DiscoveryAttributes.GroupingAttributeType.firstDate.displayName=First Activity Date",
         "DiscoveryAttributes.GroupingAttributeType.numberOfVisits.displayName=Number of Visits",
         "DiscoveryAttributes.GroupingAttributeType.none.displayName=None",
-        "DiscoveryAttributes.GroupingAttributeType.previouslyNotable.displayName=Previous Notability"})
+        "DiscoveryAttributes.GroupingAttributeType.previouslyNotable.displayName=Previous Notability",
+        "DiscoveryAttributes.GroupingAttributeType.webCategory.displayName=Domain Category"})
     public enum GroupingAttributeType {
         FILE_SIZE(new FileSizeAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_size_displayName()),
         FREQUENCY(new FrequencyAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_frequency_displayName()),
@@ -881,7 +936,8 @@ public class DiscoveryAttributes {
         FIRST_DATE(new FirstActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_firstDate_displayName()),
         NUMBER_OF_VISITS(new NumberOfVisitsAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_numberOfVisits_displayName()),
         NO_GROUPING(new NoGroupingAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_none_displayName()),
-        PREVIOUSLY_NOTABLE(new PreviouslyNotableAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_previouslyNotable_displayName());
+        PREVIOUSLY_NOTABLE(new PreviouslyNotableAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_previouslyNotable_displayName()),
+        DOMAIN_CATEGORY(new DomainCategoryAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_webCategory_displayName());
 
         private final AttributeType attributeType;
         private final String displayName;
@@ -928,7 +984,7 @@ public class DiscoveryAttributes {
          * @return Enums that can be used to group files.
          */
         public static List<GroupingAttributeType> getOptionsForGroupingForDomains() {
-            return Arrays.asList(FREQUENCY, MOST_RECENT_DATE, FIRST_DATE, NUMBER_OF_VISITS, PREVIOUSLY_NOTABLE);
+            return Arrays.asList(FREQUENCY, MOST_RECENT_DATE, FIRST_DATE, NUMBER_OF_VISITS, PREVIOUSLY_NOTABLE, DOMAIN_CATEGORY);
         }
     }
 

Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryKeyUtils.java

@@ -793,6 +793,55 @@ public class DiscoveryKeyUtils {
         }
     }
     
+    /**
+     * Group key representing a domain category (TSK_WEB_CATEGORY artifact).
+     */
+    static class DomainCategoryGroupKey extends GroupKey {
+        
+        private final String webCategory;
+        
+        DomainCategoryGroupKey(Result result) {
+            if (result instanceof ResultDomain) {
+                ResultDomain domain = (ResultDomain) result;
+                this.webCategory = domain.getWebCategory();
+            } else {
+                throw new IllegalArgumentException("Input result should be of type ResultDomain");
+            }
+        }
+
+        @Override
+        String getDisplayName() {
+            return this.webCategory;
+        }
+
+        @Override
+        public boolean equals(Object otherKey) {
+            if (otherKey instanceof GroupKey) {
+                return compareTo((GroupKey) otherKey) == 0;
+            }
+            return false;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getWebCategory());
+        }
+
+        @Override
+        public int compareTo(GroupKey otherGroupKey) {
+            if (otherGroupKey instanceof DomainCategoryGroupKey) {
+                DomainCategoryGroupKey webCategoryKey = (DomainCategoryGroupKey) otherGroupKey;
+                return this.webCategory.compareTo(webCategoryKey.getWebCategory());
+            } else {
+                return compareClassNames(otherGroupKey);
+            }
+        }
+        
+        String getWebCategory() {
+            return this.webCategory;
+        }
+    }
+    
     /**
      * Key representing a central repository notable status.
      */

Core/src/org/sleuthkit/autopsy/discovery/search/ResultDomain.java

@@ -18,6 +18,7 @@
  */
 package org.sleuthkit.autopsy.discovery.search;
 
+import org.openide.util.NbBundle;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
@@ -34,6 +35,7 @@ public class ResultDomain extends Result {
     private final Long visitsInLast60;
     private final Long filesDownloaded;
     private final Long countOfKnownAccountTypes;
+    private String webCategory;
 
     private final Content dataSource;
     private final long dataSourceId;
@@ -111,6 +113,27 @@ public class ResultDomain extends Result {
         return filesDownloaded;
     }
     
+    /**
+     * Get the web category (TSK_WEB_CATEGORY) type for this domain.
+     */
+    @NbBundle.Messages({
+        "ResultDomain_getDefaultCategory=Uncategorized"
+    })
+    public String getWebCategory() {
+        if (webCategory == null) {
+            return Bundle.ResultDomain_getDefaultCategory();
+        } else {
+            return webCategory;
+        }
+    }
+    
+    /**
+     * Set the web category for this domain (derived from TSK_WEB_CATEGORY) artifacts.
+     */
+    public void setWebCategory(String webCategory) {
+        this.webCategory = webCategory;
+    }
+    
     /**
      * Determines if the domain has been associated with a known account type
      * (TSK_WEB_ACCOUNT_TYPE).