From e33880f8a30c70f5c24e3fd98b053f39e03e21c0 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 11 Jul 2019 15:06:03 -0400 Subject: [PATCH] Added EXIF and notable filters. Added paging support. --- .../autopsy/filequery/Bundle.properties | 4 +- .../filequery/Bundle.properties-MERGED | 6 +- .../autopsy/filequery/FileGroup.java | 4 +- .../autopsy/filequery/FileSearch.java | 102 +++++++++++++++++- .../autopsy/filequery/FileSearchDialog.form | 19 ++-- .../autopsy/filequery/FileSearchDialog.java | 37 +++++-- .../filequery/FileSearchFiltering.java | 99 ++++++++++++++++- .../filequery/FileSearchTestAction.java | 55 ++++++++-- 8 files changed, 288 insertions(+), 38 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties index 20cd4e3d05..27863cffee 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties @@ -24,6 +24,6 @@ FileSearchDialog.hashCheckBox.text=Hash Set FileSearchDialog.intCheckBox.text=Interesting Items FileSearchDialog.tagsCheckBox.text=Tags FileSearchDialog.objCheckBox.text=Objects -FileSearchDialog.jCheckBox1.text=Must contain EXIF data -FileSearchDialog.jCheckBox2.text=Must have been previously tagged as notable FileSearchDialog.jCheckBox3.text=Must meet "is suspicious" criteria +FileSearchDialog.exifCheckBox.text=Must contain EXIF data +FileSearchDialog.notableCheckBox.text=Must have been tagged as notable diff --git a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED index d5cc122888..378d27edd1 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED @@ -63,6 +63,7 @@ FileSearchFiltering.DataSourceFilter.datasource={0}({1}) # {0} - filters FileSearchFiltering.DataSourceFilter.desc=Files in data source(s): {0} FileSearchFiltering.DataSourceFilter.or=\ or +FileSearchFiltering.ExifFilter.desc=Files that contain EXIF data # {0} - filters FileSearchFiltering.FileTypeFilter.desc=Files with type: {0} FileSearchFiltering.FileTypeFilter.or=\ or @@ -86,6 +87,7 @@ FileSearchFiltering.ParentFilter.substring=(substring) FileSearchFiltering.ParentSearchTerm.fullString=\ {0} (exact) # {0} - search term FileSearchFiltering.ParentSearchTerm.subString=\ {0} (substring) +FileSearchFiltering.PreviouslyNotableFilter.desc=Files that were previously marked as notable # {0} - filters FileSearchFiltering.SizeFilter.desc=Files with size in range(s): {0} FileSearchFiltering.SizeFilter.or=\ or @@ -110,6 +112,6 @@ FileSearchDialog.hashCheckBox.text=Hash Set FileSearchDialog.intCheckBox.text=Interesting Items FileSearchDialog.tagsCheckBox.text=Tags FileSearchDialog.objCheckBox.text=Objects -FileSearchDialog.jCheckBox1.text=Must contain EXIF data -FileSearchDialog.jCheckBox2.text=Must have been previously tagged as notable FileSearchDialog.jCheckBox3.text=Must meet "is suspicious" criteria +FileSearchDialog.exifCheckBox.text=Must contain EXIF data +FileSearchDialog.notableCheckBox.text=Must have been tagged as notable diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileGroup.java b/Core/src/org/sleuthkit/autopsy/filequery/FileGroup.java index 26b9b1f552..6d0ecb936c 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileGroup.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileGroup.java @@ -59,13 +59,13 @@ class FileGroup implements Comparable { } /** - * Get the display name for this group, including the size of the group. + * Get the display name for this group. * This must be unique for each group. * * @return the display name */ String getDisplayName() { - return displayName + " (" + files.size() + ")"; // NON-NLS + return displayName; // NON-NLS } /** diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java index 2c917de708..a8f8904f0c 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.filequery; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -60,7 +61,6 @@ class FileSearch { * @param groupAttributeType The attribute to use for grouping * @param groupSortingType The method to use to sort the groups * @param fileSortingMethod The method to use to sort the files within the groups - * @param attributesNeededForGroupingOrSorting Any attributes that will used for grouping or sorting * @param caseDb The case database * @param centralRepoDb The central repository database. Can be null if not needed. * @@ -73,9 +73,16 @@ class FileSearch { AttributeType groupAttributeType, FileGroup.GroupSortingAlgorithm groupSortingType, FileSorter.SortingMethod fileSortingMethod, - List attributesNeededForGroupingOrSorting, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { + // Make a list of attributes that we want to add values for. This ensures the + // ResultFile objects will have all needed fields set when it's time to group + // and sort them. For example, if we're grouping by central repo frequency, we need + // to make sure we've loaded those values before grouping. + List attributesNeededForGroupingOrSorting = new ArrayList<>(); + attributesNeededForGroupingOrSorting.add(groupAttributeType); + attributesNeededForGroupingOrSorting.addAll(fileSortingMethod.getRequiredAttributes()); + // Run the queries for each filter List resultFiles = FileSearchFiltering.runQueries(filters, caseDb, centralRepoDb); @@ -92,6 +99,87 @@ class FileSearch { return searchResults; } + /** + * Run the file search to get the group names and sizes. + * + * @param filters The filters to apply + * @param groupAttributeType The attribute to use for grouping + * @param groupSortingType The method to use to sort the groups + * @param fileSortingMethod The method to use to sort the files within the groups + * @param caseDb The case database + * @param centralRepoDb The central repository database. Can be null if not needed. + * + * @return A LinkedHashMap grouped and sorted according to the parameters + * + * @throws FileSearchException + */ + static LinkedHashMap getGroupSizes( + List filters, + AttributeType groupAttributeType, + FileGroup.GroupSortingAlgorithm groupSortingType, + FileSorter.SortingMethod fileSortingMethod, + SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { + + LinkedHashMap> searchResults = runFileSearch(filters, + groupAttributeType, groupSortingType, fileSortingMethod, caseDb, centralRepoDb); + + LinkedHashMap groupSizes = new LinkedHashMap<>(); + for (String groupName : searchResults.keySet()) { + groupSizes.put(groupName, searchResults.get(groupName).size()); + } + return groupSizes; + } + + /** + * Run the file search to get the group names and sizes. + * + * @param filters The filters to apply + * @param groupAttributeType The attribute to use for grouping + * @param groupSortingType The method to use to sort the groups + * @param fileSortingMethod The method to use to sort the files within the groups + * @param groupName Name of the group to get entries from + * @param startingEntry The first entry to return + * @param numberOfEntries The number of entries to return + * @param caseDb The case database + * @param centralRepoDb The central repository database. Can be null if not needed. + * + * @return A LinkedHashMap grouped and sorted according to the parameters + * + * @throws FileSearchException + */ + static List getGroupEntries( + List filters, + AttributeType groupAttributeType, + FileGroup.GroupSortingAlgorithm groupSortingType, + FileSorter.SortingMethod fileSortingMethod, + String groupName, + int startingEntry, + int numberOfEntries, + SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { + + LinkedHashMap> searchResults = runFileSearch(filters, + groupAttributeType, groupSortingType, fileSortingMethod, caseDb, centralRepoDb); + + List page = new ArrayList<>(); + + // Check that the group exists + if (! searchResults.containsKey(groupName)) { + return page; + } + + // Check that there is data after the starting point + if (searchResults.get(groupName).size() < startingEntry) { + return page; + } + + // Add each page in the range + for (int i = startingEntry; (i < startingEntry + numberOfEntries) + && (i < searchResults.get(groupName).size());i++) { + page.add(searchResults.get(groupName).get(i)); + } + return page; + } + /** * Run the file search. * @@ -99,7 +187,6 @@ class FileSearch { * @param groupAttributeType The attribute to use for grouping * @param groupSortingType The method to use to sort the groups * @param fileSortingMethod The method to use to sort the files within the groups - * @param attributesNeededForGroupingOrSorting Any attributes that will used for grouping or sorting * @param caseDb The case database * @param centralRepoDb The central repository database. Can be null if not needed. * @@ -112,9 +199,16 @@ class FileSearch { AttributeType groupAttributeType, FileGroup.GroupSortingAlgorithm groupSortingType, FileSorter.SortingMethod fileSortingMethod, - List attributesNeededForGroupingOrSorting, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { + // Make a list of attributes that we want to add values for. This ensures the + // ResultFile objects will have all needed fields set when it's time to group + // and sort them. For example, if we're grouping by central repo frequency, we need + // to make sure we've loaded those values before grouping. + List attributesNeededForGroupingOrSorting = new ArrayList<>(); + attributesNeededForGroupingOrSorting.add(groupAttributeType); + attributesNeededForGroupingOrSorting.addAll(fileSortingMethod.getRequiredAttributes()); + // Run the queries for each filter List resultFiles = FileSearchFiltering.runQueries(filters, caseDb, centralRepoDb); diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.form b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.form index c60b5db83f..ab99037e47 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.form +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.form @@ -120,8 +120,8 @@ - - + + @@ -192,9 +192,9 @@ - + - + @@ -703,19 +703,22 @@ - + - + - + - + + + + diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.java b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.java index f0403442da..2a7e448c2c 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchDialog.java @@ -393,6 +393,14 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe filters.add(new FileSearchFiltering.TagsFilter(tagsList.getSelectedValuesList())); } + if (exifCheckBox.isSelected()) { + filters.add(new FileSearchFiltering.ExifFilter()); + } + + if (notableCheckBox.isSelected()) { + filters.add(new FileSearchFiltering.NotableFilter()); + } + return filters; } @@ -609,8 +617,8 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe jScrollPane10 = new javax.swing.JScrollPane(); objList = new javax.swing.JList<>(); objCheckBox = new javax.swing.JCheckBox(); - jCheckBox1 = new javax.swing.JCheckBox(); - jCheckBox2 = new javax.swing.JCheckBox(); + exifCheckBox = new javax.swing.JCheckBox(); + notableCheckBox = new javax.swing.JCheckBox(); jCheckBox3 = new javax.swing.JCheckBox(); setDefaultCloseOperation(javax.swing.WindowConstants.DISPOSE_ON_CLOSE); @@ -780,9 +788,14 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe } }); - org.openide.awt.Mnemonics.setLocalizedText(jCheckBox1, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.jCheckBox1.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(exifCheckBox, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.exifCheckBox.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(jCheckBox2, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.jCheckBox2.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(notableCheckBox, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.notableCheckBox.text")); // NOI18N + notableCheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + notableCheckBoxActionPerformed(evt); + } + }); org.openide.awt.Mnemonics.setLocalizedText(jCheckBox3, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.jCheckBox3.text")); // NOI18N @@ -865,8 +878,8 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(filler2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(jCheckBox1) - .addComponent(jCheckBox2) + .addComponent(exifCheckBox) + .addComponent(notableCheckBox) .addComponent(jCheckBox3)) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)))) ); @@ -921,9 +934,9 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe .addComponent(jScrollPane5, javax.swing.GroupLayout.PREFERRED_SIZE, 49, javax.swing.GroupLayout.PREFERRED_SIZE) .addComponent(kwCheckBox) .addGroup(layout.createSequentialGroup() - .addComponent(jCheckBox1) + .addComponent(exifCheckBox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(jCheckBox2))) + .addComponent(notableCheckBox))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() @@ -1029,6 +1042,10 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe objList.setEnabled(objCheckBox.isSelected()); }//GEN-LAST:event_objCheckBoxActionPerformed + private void notableCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_notableCheckBoxActionPerformed + // TODO add your handling code here: + }//GEN-LAST:event_notableCheckBoxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton addParentButton; private javax.swing.JButton cancelButton; @@ -1036,6 +1053,7 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe private javax.swing.JCheckBox dsCheckBox; private javax.swing.JList dsList; private javax.swing.JLabel errorLabel; + private javax.swing.JCheckBox exifCheckBox; private javax.swing.JComboBox fileOrderComboBox; private javax.swing.JList fileTypeList; private javax.swing.Box.Filler filler1; @@ -1047,8 +1065,6 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe private javax.swing.JList hashList; private javax.swing.JCheckBox intCheckBox; private javax.swing.JList intList; - private javax.swing.JCheckBox jCheckBox1; - private javax.swing.JCheckBox jCheckBox2; private javax.swing.JCheckBox jCheckBox3; private javax.swing.JLabel jLabel1; private javax.swing.JLabel jLabel2; @@ -1067,6 +1083,7 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe private javax.swing.JScrollPane jScrollPane9; private javax.swing.JCheckBox kwCheckBox; private javax.swing.JList kwList; + private javax.swing.JCheckBox notableCheckBox; private javax.swing.JCheckBox objCheckBox; private javax.swing.JList objList; private javax.swing.JRadioButton orderAttrRadioButton; diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchFiltering.java b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchFiltering.java index 42d3993518..10b7574d08 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchFiltering.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchFiltering.java @@ -686,10 +686,107 @@ class FileSearchFiltering { } desc += name.getDisplayName(); } - return Bundle.FileSearchFiltering_TagsFilter_desc(desc); // Nope + return Bundle.FileSearchFiltering_TagsFilter_desc(desc); } } + /** + * A filter for specifying that the file must have EXIF data. + */ + static class ExifFilter extends FileFilter { + + /** + * Create the ExifFilter + */ + ExifFilter() { + } + + @Override + String getWhereClause() { + String queryStr = "(obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_id IN " + + "(SELECT artifact_id FROM blackboard_attributes WHERE artifact_type_id = " + + BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID() + ")))"; + + return queryStr; + } + + @NbBundle.Messages({ + "FileSearchFiltering.ExifFilter.desc=Files that contain EXIF data", + }) + @Override + String getDesc() { + return Bundle.FileSearchFiltering_ExifFilter_desc(); + } + } + + /** + * A filter for specifying that the file must have been marked as notable in the CR. + */ + static class NotableFilter extends FileFilter { + + /** + * Create the NotableFilter + */ + NotableFilter() { + } + + @Override + String getWhereClause() { + // Since this relies on the central repository database, there is no + // query on the case database. + return ""; // NON-NLS + } + + @Override + boolean useAlternateFilter() { + return true; + } + + @Override + List applyAlternateFilter (List currentResults, SleuthkitCase caseDb, + EamDb centralRepoDb) throws FileSearchException { + + if (centralRepoDb == null) { + throw new FileSearchException("Can not run Previously Notable filter with null Central Repository DB"); // NON-NLS + } + + // We have to have run some kind of SQL filter before getting to this point, + // and should have checked afterward to see if the results were empty. + if (currentResults.isEmpty()) { + throw new FileSearchException("Can not run on empty list"); // NON-NLS + } + + // The matching files + List notableResults = new ArrayList<>(); + + try { + CorrelationAttributeInstance.Type type = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(CorrelationAttributeInstance.FILES_TYPE_ID); + + for (ResultFile file : currentResults) { + if (file.getAbstractFile().getMd5Hash() != null && ! file.getAbstractFile().getMd5Hash().isEmpty()) { + + // Check if this file hash is marked as notable in the CR + String value = file.getAbstractFile().getMd5Hash(); + if (centralRepoDb.getCountArtifactInstancesKnownBad(type, value) > 0) { + notableResults.add(file); + } + } + } + return notableResults; + } catch (EamDbException | CorrelationAttributeNormalizationException ex) { + throw new FileSearchException("Error querying central repository", ex); // NON-NLS + } + } + + @NbBundle.Messages({ + "FileSearchFiltering.PreviouslyNotableFilter.desc=Files that were previously marked as notable", + }) + @Override + String getDesc() { + return Bundle.FileSearchFiltering_PreviouslyNotableFilter_desc(); + } + } + @NbBundle.Messages({ "FileSearchFiltering.concatenateSetNamesForDisplay.comma=, ", }) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchTestAction.java b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchTestAction.java index b1481dd441..90fbcafa47 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearchTestAction.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearchTestAction.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.filequery; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.logging.Level; import org.apache.commons.lang.exception.ExceptionUtils; @@ -32,6 +33,7 @@ import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; /** * Class to test the file search API. Allows the user to run searches and see results. @@ -88,21 +90,56 @@ public final class FileSearchTestAction extends CallableSystemAction { FileSorter.SortingMethod fileSort = dialog.getFileSortingMethod(); try { - - // Make a list of attributes that we want to add values for. This ensures the - // ResultFile objects will have all needed fields set when it's time to group - // and sort them. For example, if we're grouping by central repo frequency, we need - // to make sure we've loaded those values before grouping. - List attrsForGroupingAndSorting = new ArrayList<>(); - attrsForGroupingAndSorting.add(groupingAttr); - attrsForGroupingAndSorting.addAll(fileSort.getRequiredAttributes()); + + // Test getting the groups + LinkedHashMap groups = FileSearch.getGroupSizes(filters, + groupingAttr, + groupSortAlgorithm, + fileSort, + Case.getCurrentCase().getSleuthkitCase(), crDb); + + System.out.println("Groups: "); + for (String name : groups.keySet()) { + System.out.println(" " + name + " : " + groups.get(name)); + } + + if (groups.size() > 0) { + String firstGroupName = groups.keySet().iterator().next(); + List entries0to5 = FileSearch.getGroupEntries(filters, + groupingAttr, + groupSortAlgorithm, + fileSort, + firstGroupName, + 0, + 5, + Case.getCurrentCase().getSleuthkitCase(), crDb); + System.out.println("First five " + firstGroupName + " : "); + for (AbstractFile f : entries0to5) { + System.out.println(" " + f.getName()); + } + + List entries6to106 = FileSearch.getGroupEntries(filters, + groupingAttr, + groupSortAlgorithm, + fileSort, + firstGroupName, + 5, + 100, + Case.getCurrentCase().getSleuthkitCase(), crDb); + System.out.println(firstGroupName + " 6 to 106: "); + for (AbstractFile f : entries6to106) { + System.out.println(" " + f.getName()); + } + } + + + ///////////////// // Run the search SearchResults results = FileSearch.runFileSearchDebug(filters, groupingAttr, groupSortAlgorithm, fileSort, - attrsForGroupingAndSorting, Case.getCurrentCase().getSleuthkitCase(), crDb); // Display the results