Added EXIF and notable filters.

Added paging support.
This commit is contained in:
Ann Priestman 2019-07-11 15:06:03 -04:00
parent 89c059b6c3
commit e33880f8a3
8 changed files with 288 additions and 38 deletions

View File

@ -24,6 +24,6 @@ FileSearchDialog.hashCheckBox.text=Hash Set
FileSearchDialog.intCheckBox.text=Interesting Items
FileSearchDialog.tagsCheckBox.text=Tags
FileSearchDialog.objCheckBox.text=Objects
FileSearchDialog.jCheckBox1.text=Must contain EXIF data
FileSearchDialog.jCheckBox2.text=Must have been previously tagged as notable
FileSearchDialog.jCheckBox3.text=Must meet "is suspicious" criteria
FileSearchDialog.exifCheckBox.text=Must contain EXIF data
FileSearchDialog.notableCheckBox.text=Must have been tagged as notable

View File

@ -63,6 +63,7 @@ FileSearchFiltering.DataSourceFilter.datasource={0}({1})
# {0} - filters
FileSearchFiltering.DataSourceFilter.desc=Files in data source(s): {0}
FileSearchFiltering.DataSourceFilter.or=\ or
FileSearchFiltering.ExifFilter.desc=Files that contain EXIF data
# {0} - filters
FileSearchFiltering.FileTypeFilter.desc=Files with type: {0}
FileSearchFiltering.FileTypeFilter.or=\ or
@ -86,6 +87,7 @@ FileSearchFiltering.ParentFilter.substring=(substring)
FileSearchFiltering.ParentSearchTerm.fullString=\ {0} (exact)
# {0} - search term
FileSearchFiltering.ParentSearchTerm.subString=\ {0} (substring)
FileSearchFiltering.PreviouslyNotableFilter.desc=Files that were previously marked as notable
# {0} - filters
FileSearchFiltering.SizeFilter.desc=Files with size in range(s): {0}
FileSearchFiltering.SizeFilter.or=\ or
@ -110,6 +112,6 @@ FileSearchDialog.hashCheckBox.text=Hash Set
FileSearchDialog.intCheckBox.text=Interesting Items
FileSearchDialog.tagsCheckBox.text=Tags
FileSearchDialog.objCheckBox.text=Objects
FileSearchDialog.jCheckBox1.text=Must contain EXIF data
FileSearchDialog.jCheckBox2.text=Must have been previously tagged as notable
FileSearchDialog.jCheckBox3.text=Must meet "is suspicious" criteria
FileSearchDialog.exifCheckBox.text=Must contain EXIF data
FileSearchDialog.notableCheckBox.text=Must have been tagged as notable

View File

@ -59,13 +59,13 @@ class FileGroup implements Comparable<FileGroup> {
}
/**
* Get the display name for this group, including the size of the group.
* Get the display name for this group.
* This must be unique for each group.
*
* @return the display name
*/
String getDisplayName() {
return displayName + " (" + files.size() + ")"; // NON-NLS
return displayName; // NON-NLS
}
/**

View File

@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.filequery;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@ -60,7 +61,6 @@ class FileSearch {
* @param groupAttributeType The attribute to use for grouping
* @param groupSortingType The method to use to sort the groups
* @param fileSortingMethod The method to use to sort the files within the groups
* @param attributesNeededForGroupingOrSorting Any attributes that will used for grouping or sorting
* @param caseDb The case database
* @param centralRepoDb The central repository database. Can be null if not needed.
*
@ -73,9 +73,16 @@ class FileSearch {
AttributeType groupAttributeType,
FileGroup.GroupSortingAlgorithm groupSortingType,
FileSorter.SortingMethod fileSortingMethod,
List<AttributeType> attributesNeededForGroupingOrSorting,
SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException {
// Make a list of attributes that we want to add values for. This ensures the
// ResultFile objects will have all needed fields set when it's time to group
// and sort them. For example, if we're grouping by central repo frequency, we need
// to make sure we've loaded those values before grouping.
List<AttributeType> attributesNeededForGroupingOrSorting = new ArrayList<>();
attributesNeededForGroupingOrSorting.add(groupAttributeType);
attributesNeededForGroupingOrSorting.addAll(fileSortingMethod.getRequiredAttributes());
// Run the queries for each filter
List<ResultFile> resultFiles = FileSearchFiltering.runQueries(filters, caseDb, centralRepoDb);
@ -92,6 +99,87 @@ class FileSearch {
return searchResults;
}
/**
* Run the file search to get the group names and sizes.
*
* @param filters The filters to apply
* @param groupAttributeType The attribute to use for grouping
* @param groupSortingType The method to use to sort the groups
* @param fileSortingMethod The method to use to sort the files within the groups
* @param caseDb The case database
* @param centralRepoDb The central repository database. Can be null if not needed.
*
* @return A LinkedHashMap grouped and sorted according to the parameters
*
* @throws FileSearchException
*/
static LinkedHashMap<String, Integer> getGroupSizes(
List<FileSearchFiltering.FileFilter> filters,
AttributeType groupAttributeType,
FileGroup.GroupSortingAlgorithm groupSortingType,
FileSorter.SortingMethod fileSortingMethod,
SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException {
LinkedHashMap<String, List<AbstractFile>> searchResults = runFileSearch(filters,
groupAttributeType, groupSortingType, fileSortingMethod, caseDb, centralRepoDb);
LinkedHashMap<String, Integer> groupSizes = new LinkedHashMap<>();
for (String groupName : searchResults.keySet()) {
groupSizes.put(groupName, searchResults.get(groupName).size());
}
return groupSizes;
}
/**
* Run the file search to get the group names and sizes.
*
* @param filters The filters to apply
* @param groupAttributeType The attribute to use for grouping
* @param groupSortingType The method to use to sort the groups
* @param fileSortingMethod The method to use to sort the files within the groups
* @param groupName Name of the group to get entries from
* @param startingEntry The first entry to return
* @param numberOfEntries The number of entries to return
* @param caseDb The case database
* @param centralRepoDb The central repository database. Can be null if not needed.
*
* @return A LinkedHashMap grouped and sorted according to the parameters
*
* @throws FileSearchException
*/
static List<AbstractFile> getGroupEntries(
List<FileSearchFiltering.FileFilter> filters,
AttributeType groupAttributeType,
FileGroup.GroupSortingAlgorithm groupSortingType,
FileSorter.SortingMethod fileSortingMethod,
String groupName,
int startingEntry,
int numberOfEntries,
SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException {
LinkedHashMap<String, List<AbstractFile>> searchResults = runFileSearch(filters,
groupAttributeType, groupSortingType, fileSortingMethod, caseDb, centralRepoDb);
List<AbstractFile> page = new ArrayList<>();
// Check that the group exists
if (! searchResults.containsKey(groupName)) {
return page;
}
// Check that there is data after the starting point
if (searchResults.get(groupName).size() < startingEntry) {
return page;
}
// Add each page in the range
for (int i = startingEntry; (i < startingEntry + numberOfEntries)
&& (i < searchResults.get(groupName).size());i++) {
page.add(searchResults.get(groupName).get(i));
}
return page;
}
/**
* Run the file search.
*
@ -99,7 +187,6 @@ class FileSearch {
* @param groupAttributeType The attribute to use for grouping
* @param groupSortingType The method to use to sort the groups
* @param fileSortingMethod The method to use to sort the files within the groups
* @param attributesNeededForGroupingOrSorting Any attributes that will used for grouping or sorting
* @param caseDb The case database
* @param centralRepoDb The central repository database. Can be null if not needed.
*
@ -112,9 +199,16 @@ class FileSearch {
AttributeType groupAttributeType,
FileGroup.GroupSortingAlgorithm groupSortingType,
FileSorter.SortingMethod fileSortingMethod,
List<AttributeType> attributesNeededForGroupingOrSorting,
SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException {
// Make a list of attributes that we want to add values for. This ensures the
// ResultFile objects will have all needed fields set when it's time to group
// and sort them. For example, if we're grouping by central repo frequency, we need
// to make sure we've loaded those values before grouping.
List<AttributeType> attributesNeededForGroupingOrSorting = new ArrayList<>();
attributesNeededForGroupingOrSorting.add(groupAttributeType);
attributesNeededForGroupingOrSorting.addAll(fileSortingMethod.getRequiredAttributes());
// Run the queries for each filter
List<ResultFile> resultFiles = FileSearchFiltering.runQueries(filters, caseDb, centralRepoDb);

View File

@ -120,8 +120,8 @@
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="filler2" min="-2" max="-2" attributes="0"/>
<Component id="jCheckBox1" min="-2" max="-2" attributes="0"/>
<Component id="jCheckBox2" min="-2" max="-2" attributes="0"/>
<Component id="exifCheckBox" min="-2" max="-2" attributes="0"/>
<Component id="notableCheckBox" min="-2" max="-2" attributes="0"/>
<Component id="jCheckBox3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="32767" attributes="0"/>
@ -192,9 +192,9 @@
<Component id="jScrollPane5" min="-2" pref="49" max="-2" attributes="0"/>
<Component id="kwCheckBox" min="-2" max="-2" attributes="0"/>
<Group type="102" attributes="0">
<Component id="jCheckBox1" min="-2" max="-2" attributes="0"/>
<Component id="exifCheckBox" min="-2" max="-2" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
<Component id="jCheckBox2" min="-2" max="-2" attributes="0"/>
<Component id="notableCheckBox" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
@ -703,19 +703,22 @@
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="objCheckBoxActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JCheckBox" name="jCheckBox1">
<Component class="javax.swing.JCheckBox" name="exifCheckBox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/filequery/Bundle.properties" key="FileSearchDialog.jCheckBox1.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/filequery/Bundle.properties" key="FileSearchDialog.exifCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JCheckBox" name="jCheckBox2">
<Component class="javax.swing.JCheckBox" name="notableCheckBox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/filequery/Bundle.properties" key="FileSearchDialog.jCheckBox2.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
<ResourceString bundle="org/sleuthkit/autopsy/filequery/Bundle.properties" key="FileSearchDialog.notableCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="notableCheckBoxActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JCheckBox" name="jCheckBox3">
<Properties>

View File

@ -393,6 +393,14 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
filters.add(new FileSearchFiltering.TagsFilter(tagsList.getSelectedValuesList()));
}
if (exifCheckBox.isSelected()) {
filters.add(new FileSearchFiltering.ExifFilter());
}
if (notableCheckBox.isSelected()) {
filters.add(new FileSearchFiltering.NotableFilter());
}
return filters;
}
@ -609,8 +617,8 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
jScrollPane10 = new javax.swing.JScrollPane();
objList = new javax.swing.JList<>();
objCheckBox = new javax.swing.JCheckBox();
jCheckBox1 = new javax.swing.JCheckBox();
jCheckBox2 = new javax.swing.JCheckBox();
exifCheckBox = new javax.swing.JCheckBox();
notableCheckBox = new javax.swing.JCheckBox();
jCheckBox3 = new javax.swing.JCheckBox();
setDefaultCloseOperation(javax.swing.WindowConstants.DISPOSE_ON_CLOSE);
@ -780,9 +788,14 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
}
});
org.openide.awt.Mnemonics.setLocalizedText(jCheckBox1, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.jCheckBox1.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(exifCheckBox, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.exifCheckBox.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(jCheckBox2, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.jCheckBox2.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(notableCheckBox, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.notableCheckBox.text")); // NOI18N
notableCheckBox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
notableCheckBoxActionPerformed(evt);
}
});
org.openide.awt.Mnemonics.setLocalizedText(jCheckBox3, org.openide.util.NbBundle.getMessage(FileSearchDialog.class, "FileSearchDialog.jCheckBox3.text")); // NOI18N
@ -865,8 +878,8 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(filler2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(jCheckBox1)
.addComponent(jCheckBox2)
.addComponent(exifCheckBox)
.addComponent(notableCheckBox)
.addComponent(jCheckBox3))
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
);
@ -921,9 +934,9 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
.addComponent(jScrollPane5, javax.swing.GroupLayout.PREFERRED_SIZE, 49, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(kwCheckBox)
.addGroup(layout.createSequentialGroup()
.addComponent(jCheckBox1)
.addComponent(exifCheckBox)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(jCheckBox2)))
.addComponent(notableCheckBox)))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
@ -1029,6 +1042,10 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
objList.setEnabled(objCheckBox.isSelected());
}//GEN-LAST:event_objCheckBoxActionPerformed
private void notableCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_notableCheckBoxActionPerformed
// TODO add your handling code here:
}//GEN-LAST:event_notableCheckBoxActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton addParentButton;
private javax.swing.JButton cancelButton;
@ -1036,6 +1053,7 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
private javax.swing.JCheckBox dsCheckBox;
private javax.swing.JList<DataSourceItem> dsList;
private javax.swing.JLabel errorLabel;
private javax.swing.JCheckBox exifCheckBox;
private javax.swing.JComboBox<SortingMethod> fileOrderComboBox;
private javax.swing.JList<FileSearchData.FileType> fileTypeList;
private javax.swing.Box.Filler filler1;
@ -1047,8 +1065,6 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
private javax.swing.JList<String> hashList;
private javax.swing.JCheckBox intCheckBox;
private javax.swing.JList<String> intList;
private javax.swing.JCheckBox jCheckBox1;
private javax.swing.JCheckBox jCheckBox2;
private javax.swing.JCheckBox jCheckBox3;
private javax.swing.JLabel jLabel1;
private javax.swing.JLabel jLabel2;
@ -1067,6 +1083,7 @@ public class FileSearchDialog extends javax.swing.JDialog implements ActionListe
private javax.swing.JScrollPane jScrollPane9;
private javax.swing.JCheckBox kwCheckBox;
private javax.swing.JList<String> kwList;
private javax.swing.JCheckBox notableCheckBox;
private javax.swing.JCheckBox objCheckBox;
private javax.swing.JList<String> objList;
private javax.swing.JRadioButton orderAttrRadioButton;

View File

@ -686,10 +686,107 @@ class FileSearchFiltering {
}
desc += name.getDisplayName();
}
return Bundle.FileSearchFiltering_TagsFilter_desc(desc); // Nope
return Bundle.FileSearchFiltering_TagsFilter_desc(desc);
}
}
/**
* A filter for specifying that the file must have EXIF data.
*/
static class ExifFilter extends FileFilter {
/**
* Create the ExifFilter
*/
ExifFilter() {
}
@Override
String getWhereClause() {
String queryStr = "(obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_id IN " +
"(SELECT artifact_id FROM blackboard_attributes WHERE artifact_type_id = " +
BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID() + ")))";
return queryStr;
}
@NbBundle.Messages({
"FileSearchFiltering.ExifFilter.desc=Files that contain EXIF data",
})
@Override
String getDesc() {
return Bundle.FileSearchFiltering_ExifFilter_desc();
}
}
/**
* A filter for specifying that the file must have been marked as notable in the CR.
*/
static class NotableFilter extends FileFilter {
/**
* Create the NotableFilter
*/
NotableFilter() {
}
@Override
String getWhereClause() {
// Since this relies on the central repository database, there is no
// query on the case database.
return ""; // NON-NLS
}
@Override
boolean useAlternateFilter() {
return true;
}
@Override
List<ResultFile> applyAlternateFilter (List<ResultFile> currentResults, SleuthkitCase caseDb,
EamDb centralRepoDb) throws FileSearchException {
if (centralRepoDb == null) {
throw new FileSearchException("Can not run Previously Notable filter with null Central Repository DB"); // NON-NLS
}
// We have to have run some kind of SQL filter before getting to this point,
// and should have checked afterward to see if the results were empty.
if (currentResults.isEmpty()) {
throw new FileSearchException("Can not run on empty list"); // NON-NLS
}
// The matching files
List<ResultFile> notableResults = new ArrayList<>();
try {
CorrelationAttributeInstance.Type type = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(CorrelationAttributeInstance.FILES_TYPE_ID);
for (ResultFile file : currentResults) {
if (file.getAbstractFile().getMd5Hash() != null && ! file.getAbstractFile().getMd5Hash().isEmpty()) {
// Check if this file hash is marked as notable in the CR
String value = file.getAbstractFile().getMd5Hash();
if (centralRepoDb.getCountArtifactInstancesKnownBad(type, value) > 0) {
notableResults.add(file);
}
}
}
return notableResults;
} catch (EamDbException | CorrelationAttributeNormalizationException ex) {
throw new FileSearchException("Error querying central repository", ex); // NON-NLS
}
}
@NbBundle.Messages({
"FileSearchFiltering.PreviouslyNotableFilter.desc=Files that were previously marked as notable",
})
@Override
String getDesc() {
return Bundle.FileSearchFiltering_PreviouslyNotableFilter_desc();
}
}
@NbBundle.Messages({
"FileSearchFiltering.concatenateSetNamesForDisplay.comma=, ",
})

View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.filequery;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.logging.Level;
import org.apache.commons.lang.exception.ExceptionUtils;
@ -32,6 +33,7 @@ import org.openide.util.NbBundle;
import org.openide.util.actions.CallableSystemAction;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Class to test the file search API. Allows the user to run searches and see results.
@ -88,21 +90,56 @@ public final class FileSearchTestAction extends CallableSystemAction {
FileSorter.SortingMethod fileSort = dialog.getFileSortingMethod();
try {
// Make a list of attributes that we want to add values for. This ensures the
// ResultFile objects will have all needed fields set when it's time to group
// and sort them. For example, if we're grouping by central repo frequency, we need
// to make sure we've loaded those values before grouping.
List<FileSearch.AttributeType> attrsForGroupingAndSorting = new ArrayList<>();
attrsForGroupingAndSorting.add(groupingAttr);
attrsForGroupingAndSorting.addAll(fileSort.getRequiredAttributes());
// Test getting the groups
LinkedHashMap<String, Integer> groups = FileSearch.getGroupSizes(filters,
groupingAttr,
groupSortAlgorithm,
fileSort,
Case.getCurrentCase().getSleuthkitCase(), crDb);
System.out.println("Groups: ");
for (String name : groups.keySet()) {
System.out.println(" " + name + " : " + groups.get(name));
}
if (groups.size() > 0) {
String firstGroupName = groups.keySet().iterator().next();
List<AbstractFile> entries0to5 = FileSearch.getGroupEntries(filters,
groupingAttr,
groupSortAlgorithm,
fileSort,
firstGroupName,
0,
5,
Case.getCurrentCase().getSleuthkitCase(), crDb);
System.out.println("First five " + firstGroupName + " : ");
for (AbstractFile f : entries0to5) {
System.out.println(" " + f.getName());
}
List<AbstractFile> entries6to106 = FileSearch.getGroupEntries(filters,
groupingAttr,
groupSortAlgorithm,
fileSort,
firstGroupName,
5,
100,
Case.getCurrentCase().getSleuthkitCase(), crDb);
System.out.println(firstGroupName + " 6 to 106: ");
for (AbstractFile f : entries6to106) {
System.out.println(" " + f.getName());
}
}
/////////////////
// Run the search
SearchResults results = FileSearch.runFileSearchDebug(filters,
groupingAttr,
groupSortAlgorithm,
fileSort,
attrsForGroupingAndSorting,
Case.getCurrentCase().getSleuthkitCase(), crDb);
// Display the results