Extended tag display and filtering to keyword and hash set hits and prohibited making local copies of unalloc/virt files for reports

Core/src/org/sleuthkit/autopsy/datamodel/Tags.java

@@ -611,33 +611,47 @@ public class Tags implements AutopsyVisitableItem {
     }
 
     /**
-     * Looks up the tag names associated with an artifact.
+     * Looks up the tag names associated with either a tagged artifact or a tag artifact.
      * 
      * @param artifact The artifact
      * @return A set of unique tag names
      */
     public static HashSet<String> getUniqueTagNames(BlackboardArtifact artifact) {
-        HashSet<String> tagNames = new HashSet<>();
+        return getUniqueTagNames(artifact.getArtifactID(), artifact.getArtifactTypeID());
-
-        List<BlackboardArtifact> tags;
-        try {
-            if (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_TAG_FILE.getTypeID() ||
-                artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_TAG_ARTIFACT.getTypeID()) {
-                tags = new ArrayList<>();
-                tags.add(artifact);
-            } else {
-                tags = Case.getCurrentCase().getSleuthkitCase().getBlackboardArtifacts(ATTRIBUTE_TYPE.TSK_TAGGED_ARTIFACT, artifact.getArtifactID());
     }    
 
+    /**
+     * Looks up the tag names associated with either a tagged artifact or a tag artifact.
+     * 
+     * @param artifactID The ID of the artifact
+     * @param artifactTypeID The ID of the artifact type
+     * @return A set of unique tag names
+     */
+    public static HashSet<String> getUniqueTagNames(long artifactID, int artifactTypeID) {
+        HashSet<String> tagNames = new HashSet<>();
+        
+        try {
+            ArrayList<Long> tagArtifactIDs = new ArrayList<>();
+            if (artifactTypeID == ARTIFACT_TYPE.TSK_TAG_FILE.getTypeID() ||
+                artifactTypeID == ARTIFACT_TYPE.TSK_TAG_ARTIFACT.getTypeID()) {
+                tagArtifactIDs.add(artifactID);
+            } else {
+                List<BlackboardArtifact> tags = Case.getCurrentCase().getSleuthkitCase().getBlackboardArtifacts(ATTRIBUTE_TYPE.TSK_TAGGED_ARTIFACT, artifactID);
                 for (BlackboardArtifact tag : tags) {
-                String whereClause = "WHERE artifact_id=" + tag.getArtifactID() + " AND attribute_type_id=" + ATTRIBUTE_TYPE.TSK_TAG_NAME.getTypeID();
+                    tagArtifactIDs.add(tag.getArtifactID());
+                }
+            }
+
+            for (Long tagArtifactID : tagArtifactIDs) {
+                String whereClause = "WHERE artifact_id = " + tagArtifactID + " AND attribute_type_id = " + ATTRIBUTE_TYPE.TSK_TAG_NAME.getTypeID();
                 List<BlackboardAttribute> attributes = Case.getCurrentCase().getSleuthkitCase().getMatchingAttributes(whereClause);
                 for (BlackboardAttribute attr : attributes) {
                     tagNames.add(attr.getValueString());
                 }
             }
-        } catch (TskCoreException ex) {
+        } 
-            logger.log(Level.SEVERE, "Failed to get tags for artifact " + artifact.getArtifactID(), ex);
+        catch (TskCoreException ex) {
+            logger.log(Level.SEVERE, "Failed to get tags for artifact " + artifactID, ex);
         }
         
         return tagNames;

Core/src/org/sleuthkit/autopsy/report/ReportGenerator.java

@@ -97,8 +97,8 @@ public class ReportGenerator {
         }
         
         // Initialize the progress panels
-        generalProgress = new HashMap<GeneralReportModule, ReportProgressPanel>();
+        generalProgress = new HashMap<>();
-        tableProgress = new HashMap<TableReportModule, ReportProgressPanel>();
+        tableProgress = new HashMap<>();
         setupProgressPanels(tableModuleStates, generalModuleStates);
     }
     
@@ -171,7 +171,7 @@ public class ReportGenerator {
      * @param tagSelections the enabled/disabled state of the tags to be included in the report
      */
     public void generateArtifactTableReports(Map<ARTIFACT_TYPE, Boolean> artifactTypeSelections, Map<String, Boolean> tagSelections) {
-        ArtifactTableReportsWorker worker = new ArtifactTableReportsWorker(artifactTypeSelections, tagSelections);
+        ArtifactsReportsWorker worker = new ArtifactsReportsWorker(artifactTypeSelections, tagSelections);
         worker.execute();
     }
     
@@ -194,21 +194,21 @@ public class ReportGenerator {
     }
     
     /**
-     * SwingWorker to use TableReportModules to generate reports on blackboard artifacts.
+     * SwingWorker to generate reports on blackboard artifacts.
      */
-    private class ArtifactTableReportsWorker extends SwingWorker<Integer, Integer> {
+    private class ArtifactsReportsWorker extends SwingWorker<Integer, Integer> {
         List<TableReportModule> tableModules;
         List<ARTIFACT_TYPE> artifactTypes;
         HashSet<String> tagNamesFilter;
         
         // Create an ArtifactWorker with the enabled/disabled state of all Artifacts
-        ArtifactTableReportsWorker(Map<ARTIFACT_TYPE, Boolean> artifactTypeSelections, Map<String, Boolean> tagSelections) {
+        ArtifactsReportsWorker(Map<ARTIFACT_TYPE, Boolean> artifactTypeSelections, Map<String, Boolean> tagSelections) {
-            tableModules = new ArrayList<TableReportModule>();
+            tableModules = new ArrayList<>();
             for (Entry<TableReportModule, ReportProgressPanel> entry : tableProgress.entrySet()) {
                 tableModules.add(entry.getKey());
             }
             
-            artifactTypes = new ArrayList<ARTIFACT_TYPE>();
+            artifactTypes = new ArrayList<>();
             for (Entry<ARTIFACT_TYPE, Boolean> entry : artifactTypeSelections.entrySet()) {
                 if (entry.getValue()) {
                     artifactTypes.add(entry.getKey());
@@ -216,7 +216,7 @@ public class ReportGenerator {
             }
             
             if (tagSelections != null) {
-                tagNamesFilter = new HashSet<String>();
+                tagNamesFilter = new HashSet<>();
                 for (Entry<String, Boolean> entry : tagSelections.entrySet()) {
                     if (entry.getValue() == true) {
                         tagNamesFilter.add(entry.getKey());
@@ -254,10 +254,10 @@ public class ReportGenerator {
                 
                 // If the type is keyword hit or hashset hit, use the helper
                 if (type.equals(ARTIFACT_TYPE.TSK_KEYWORD_HIT)) {
-                    writeKeywordHits(tableModules);
+                    writeKeywordHits(tableModules, tagNamesFilter);
                     continue;
                 } else if (type.equals(ARTIFACT_TYPE.TSK_HASHSET_HIT)) {
-                    writeHashsetHits(tableModules);
+                    writeHashsetHits(tableModules, tagNamesFilter);
                     continue;
                 }
 
@@ -292,7 +292,6 @@ public class ReportGenerator {
                 
                     module.startDataType(type.getDisplayName());                        
                 
-                    // This is a temporary expedient pending modification of the TableReportModule API.
                     if (module instanceof ReportHTML) {
                         ReportHTML htmlReportModule = (ReportHTML)module;
                         htmlReportModule.startTable(columnHeaders, type);
@@ -306,13 +305,9 @@ public class ReportGenerator {
                 for (Entry<BlackboardArtifact, List<BlackboardAttribute>> artifactEntry : unsortedArtifacts) {                    
                     // Get any tags associated with the artifact and apply the tags filter, if any.
                     HashSet<String> tags = Tags.getUniqueTagNames(artifactEntry.getKey());
-                    if (tagNamesFilter != null) {
+                    if (failsTagFilter(tags, tagNamesFilter)) {
-                        HashSet<String> filteredTags = new HashSet<>(tags);
-                        filteredTags.retainAll(tagNamesFilter);
-                        if (filteredTags.isEmpty()) {
                         continue;
                     }                    
-                    }                    
                     String tagsList = makeCommaSeparatedList(tags);
                     
                     // Add the row data to all of the reports.
@@ -328,7 +323,6 @@ public class ReportGenerator {
                             rowData.add(tagsList);
                         }
 
-                        // This is a temporary expedient pending modification of the TableReportModule API.
                         if (module instanceof ReportHTML) {
                             ReportHTML htmlReportModule = (ReportHTML)module;
                             htmlReportModule.addRow(rowData, artifactEntry.getKey());
@@ -357,12 +351,23 @@ public class ReportGenerator {
         }
     }
     
+    private Boolean failsTagFilter(HashSet<String> tags, HashSet<String> tagsFilter) 
+    {
+        if (tagsFilter == null) {
+            return false;
+        }
+
+        HashSet<String> filteredTags = new HashSet<>(tags);
+        filteredTags.retainAll(tagsFilter);
+        return filteredTags.isEmpty();
+    }
+            
     /**
      * Write the keyword hits to the provided TableReportModules.
      * @param tableModules modules to report on
      */
     @SuppressWarnings("deprecation")
-    private void writeKeywordHits(List<TableReportModule> tableModules) {
+    private void writeKeywordHits(List<TableReportModule> tableModules, HashSet<String> tagNamesFilter) {
         ResultSet listsRs = null;
         try {
             // Query for keyword lists
@@ -403,7 +408,7 @@ public class ReportGenerator {
         ResultSet rs = null;
         try {
             // Query for keywords
-            rs = skCase.runQuery("SELECT art.obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name " +
+            rs = skCase.runQuery("SELECT art.artifact_id, art.obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name " +
                                            "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, blackboard_attributes AS att3, tsk_files AS f " +
                                            "WHERE (att1.artifact_id = art.artifact_id) " +
                                                  "AND (att2.artifact_id = art.artifact_id) " + 
@@ -429,6 +434,13 @@ public class ReportGenerator {
                     }
                 }
  
+               // Get any tags that associated with this artifact and apply the tag filter.
+                HashSet<String> tags = Tags.getUniqueTagNames(rs.getLong("artifact_id"), ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID());
+                if (failsTagFilter(tags, tagNamesFilter)) {
+                    continue;
+                }                    
+                String tagsList = makeCommaSeparatedList(tags);
+                                                        
                 Long objId = rs.getLong("obj_id");
                 String keyword = rs.getString("keyword");
                 String preview = rs.getString("preview");
@@ -470,9 +482,10 @@ public class ReportGenerator {
                         module.startTable(getArtifactTableColumnHeaders(ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()));
                     }
                 }
+                
                 String previewreplace = EscapeUtil.escapeHtml(preview);
                 for (TableReportModule module : tableModules) {
-                    module.addRow(Arrays.asList(new String[] {previewreplace.replaceAll("<!", ""), uniquePath}));
+                    module.addRow(Arrays.asList(new String[] {previewreplace.replaceAll("<!", ""), uniquePath, tagsList}));
                 }
             }
             
@@ -498,7 +511,7 @@ public class ReportGenerator {
      * @param tableModules modules to report on
      */
     @SuppressWarnings("deprecation")
-    private void writeHashsetHits(List<TableReportModule> tableModules) {
+    private void writeHashsetHits(List<TableReportModule> tableModules, HashSet<String> tagNamesFilter) {
         ResultSet listsRs = null;
         try {
             // Query for hashsets
@@ -534,7 +547,7 @@ public class ReportGenerator {
         ResultSet rs = null;
         try {
             // Query for hashset hits
-            rs = skCase.runQuery("SELECT art.obj_id, att.value_text AS setname, f.name AS name, f.size AS size " +
+            rs = skCase.runQuery("SELECT art.artifact_id, art.obj_id, att.value_text AS setname, f.name AS name, f.size AS size " +
                                            "FROM blackboard_artifacts AS art, blackboard_attributes AS att, tsk_files AS f " +
                                            "WHERE (att.artifact_id = art.artifact_id) " +
                                                  "AND (f.obj_id = art.obj_id) " +
@@ -555,6 +568,13 @@ public class ReportGenerator {
                     }
                 }
                 
+               // Get any tags that associated with this artifact and apply the tag filter.
+                HashSet<String> tags = Tags.getUniqueTagNames(rs.getLong("artifact_id"), ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID());
+                if (failsTagFilter(tags, tagNamesFilter)) {
+                    continue;
+                }                    
+                String tagsList = makeCommaSeparatedList(tags);
+                                                                        
                 Long objId = rs.getLong("obj_id");
                 String set = rs.getString("setname");
                 String size = rs.getString("size");
@@ -586,7 +606,7 @@ public class ReportGenerator {
                 
                 // Add a row for this hit to every module
                 for (TableReportModule module : tableModules) {
-                    module.addRow(Arrays.asList(new String[] {uniquePath, size}));
+                    module.addRow(Arrays.asList(new String[] {uniquePath, size, tagsList}));
                 }
             }
             
@@ -661,9 +681,7 @@ public class ReportGenerator {
         }
 
         if (artifactTypeId != ARTIFACT_TYPE.TSK_TAG_FILE.getTypeID() && 
-            artifactTypeId != ARTIFACT_TYPE.TSK_TAG_ARTIFACT.getTypeID() &&
+            artifactTypeId != ARTIFACT_TYPE.TSK_TAG_ARTIFACT.getTypeID()) {
-            artifactTypeId != ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() &&
-            artifactTypeId != ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID()) {
             columnHeaders.add("Tags");
         }
         

Core/src/org/sleuthkit/autopsy/report/ReportHTML.java

@@ -55,6 +55,8 @@ import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.BlackboardArtifact;
 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
 import org.sleuthkit.datamodel.AbstractFile;
+import org.sleuthkit.datamodel.TskData;
+import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM;
 
 public class ReportHTML implements TableReportModule {
     private static final Logger logger = Logger.getLogger(ReportHTML.class.getName());
@@ -144,7 +146,7 @@ public class ReportHTML implements TableReportModule {
 
     /**
      * Start a new HTML page for the given data type. Update the output stream to this page,
-     * and setup the webpage header.
+     * and setup the web page header.
      * @param title title of the data type
      */
     @Override
@@ -176,7 +178,7 @@ public class ReportHTML implements TableReportModule {
     }
     
     /**
-     * End the current data type. Write the end of the webpage and close the
+     * End the current data type. Write the end of the web page and close the
      * output stream.
      */
     @Override
@@ -282,6 +284,7 @@ public class ReportHTML implements TableReportModule {
 
     /**
      * Start a new table with the given column headers.
+     * 
      * @param columnHeaders column headers
      * @param sourceArtifact source blackboard artifact for the table data 
      */
@@ -294,7 +297,7 @@ public class ReportHTML implements TableReportModule {
         
         // For file tag artifacts, add a column for a hyperlink to a local copy of the tagged file.            
         if (artifactType.equals(ARTIFACT_TYPE.TSK_TAG_FILE)) {
-            htmlOutput.append("\t\t<th>Local File</th>\n");        
+            htmlOutput.append("\t\t<th></th>\n");        
         }
         
         htmlOutput.append("\t</tr>\n</thead>\n");
@@ -348,21 +351,75 @@ public class ReportHTML implements TableReportModule {
      * @param sourceArtifact source blackboard artifact for the table data 
      */
     public void addRow(List<String> row, BlackboardArtifact sourceArtifact) {
-        // For file tag artifacts, save a local copy of the tagged file and include a hyperlink to it in the row.
+        addRowDataForSourceArtifact(row, sourceArtifact);
-        if (sourceArtifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_TAG_FILE.getTypeID()) {
+        
+        StringBuilder builder = new StringBuilder();
+        builder.append("\t<tr>\n");
+        for (String cell : row) {
+            builder.append("\t\t<td>").append(cell).append("</td>\n");
+        }
+        builder.append("\t</tr>\n");
+        rowCount++;
+        
         try {
+            out.write(builder.toString());
+        } 
+        catch (IOException ex) {
+            logger.log(Level.SEVERE, "Failed to write row to out.", ex);
+        } 
+        catch (NullPointerException ex) {
+            logger.log(Level.SEVERE, "Output writer is null. Page was not initialized before writing.", ex);
+        }
+    }
+        
+    /**
+     * Add cells particular to a type of artifact associated with the row. Assumes that the overload of startTable() that takes an artifact type was called.
+     * 
+     * @param row The row.
+     * @param sourceArtifact The artifact associated with the row. 
+     */
+    private void addRowDataForSourceArtifact(List<String> row, BlackboardArtifact sourceArtifact) {
+        int artifactTypeID = sourceArtifact.getArtifactTypeID();
+        switch (artifactTypeID) {
+            case 17:
+                addRowDataForFileTagArtifact(row, sourceArtifact);                
+                break;
+            default:
+                break;
+        }
+    }
+
+    /**
+     * Saves a local copy of a tagged file and adds a hyper link to the file to the row.
+     * 
+     * @param row The row.
+     * @param sourceArtifact The artifact associated with the row. 
+     */
+    private void addRowDataForFileTagArtifact(List<String> row, BlackboardArtifact sourceArtifact) {
+        try {
+            AbstractFile file = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(sourceArtifact.getObjectID());                
+
+            // Don't make a local copy of the file if it is unallocated space or a virtual directory.
+            if (file.getType() == TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS ||
+                file.getType() == TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS ||
+                file.getType() == TSK_DB_FILES_TYPE_ENUM.VIRTUAL_DIR) {
+                row.add("");
+                return;
+            }
+            
             // Make a folder for the local file with the same name as the tag.
             StringBuilder localFilePath = new StringBuilder();
             localFilePath.append(path);
             HashSet<String> tagNames = Tags.getUniqueTagNames(sourceArtifact);
+            if (!tagNames.isEmpty()) {
                 localFilePath.append(tagNames.iterator().next());
-                File tagFolder = new File(localFilePath.toString());
+            }
-                if (!tagFolder.exists()) { 
+            File localFileFolder = new File(localFilePath.toString());
-                    tagFolder.mkdirs();
+            if (!localFileFolder.exists()) { 
+                localFileFolder.mkdirs();
             }
 
             // Construct a file name for the local file that incorporates the corresponding object id to ensure uniqueness.
-                AbstractFile file = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(sourceArtifact.getObjectID());                
             String fileName = file.getName();
             String objectIdSuffix = "_" + sourceArtifact.getObjectID();
             int lastDotIndex = fileName.lastIndexOf(".");
@@ -389,9 +446,7 @@ public class ReportHTML implements TableReportModule {
             StringBuilder localFileLink = new StringBuilder();
             localFileLink.append("<a href=\"file:///");
             localFileLink.append(localFilePath.toString());
-                localFileLink.append("\">");
+            localFileLink.append("\">View File</a>");
-                localFileLink.append(localFilePath.toString());
-                localFileLink.append("</a>");
             row.add(localFileLink.toString());              
         } 
         catch (TskCoreException ex) {
@@ -400,25 +455,6 @@ public class ReportHTML implements TableReportModule {
         }                                    
     }
     
-        StringBuilder builder = new StringBuilder();
-        builder.append("\t<tr>\n");
-        for (String cell : row) {
-            builder.append("\t\t<td>").append(cell).append("</td>\n");
-        }
-        builder.append("\t</tr>\n");
-        rowCount++;
-        
-        try {
-            out.write(builder.toString());
-        } 
-        catch (IOException ex) {
-            logger.log(Level.SEVERE, "Failed to write row to out.", ex);
-        } 
-        catch (NullPointerException ex) {
-            logger.log(Level.SEVERE, "Output writer is null. Page was not initialized before writing.", ex);
-        }
-    }
-        
     /**
      * Return a String date for the long date given.
      * @param date date as a long