diff --git a/Core/src/org/sleuthkit/autopsy/newpackage/FileGroup.java b/Core/src/org/sleuthkit/autopsy/newpackage/FileGroup.java index 1fdb148787..145f327d3f 100644 --- a/Core/src/org/sleuthkit/autopsy/newpackage/FileGroup.java +++ b/Core/src/org/sleuthkit/autopsy/newpackage/FileGroup.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.newpackage; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; @@ -32,8 +31,6 @@ import org.sleuthkit.datamodel.AbstractFile; class FileGroup implements Comparable { private final FileGroup.GroupSortingAlgorithm groupSortingType; - //private final FileSearch.AttributeType attrType; - private final Comparator fileSortingMethod; private final FileSearch.GroupKey groupKey; private final List files; private final String displayName; @@ -41,17 +38,11 @@ class FileGroup implements Comparable { /** * Create a FileGroup object with its first file. * - * @param attrType The type of attribute being used for grouping * @param groupSortingType The method for sorting the group - * @param fileSortingMethod The method for sorting files within the group * @param groupKey The GroupKey for this group */ - FileGroup(//FileSearch.AttributeType attrType, - FileGroup.GroupSortingAlgorithm groupSortingType, - Comparator fileSortingMethod, FileSearch.GroupKey groupKey) { + FileGroup(FileGroup.GroupSortingAlgorithm groupSortingType, FileSearch.GroupKey groupKey) { this.groupSortingType = groupSortingType; - //this.attrType = attrType; - this.fileSortingMethod = fileSortingMethod; this.groupKey = groupKey; files = new ArrayList<>(); this.displayName = groupKey.getDisplayName(); @@ -89,8 +80,8 @@ class FileGroup implements Comparable { /** * Sort all the files in the group */ - void sortFiles() { - Collections.sort(files, fileSortingMethod); + void sortFiles(FileSorter sorter) { + Collections.sort(files, sorter); } /** @@ -116,28 +107,27 @@ class FileGroup implements Comparable { switch (groupSortingType) { case BY_GROUP_SIZE: return compareGroupsBySize(this, otherGroup); - case BY_ATTRIBUTE: + case BY_GROUP_KEY: default: - return compareGroupsByAttribute(this, otherGroup); + return compareGroupsByGroupKey(this, otherGroup); } } /** - * Compare two groups based on the grouping attribute. + * Compare two groups based on the group key * * @param group1 * @param group2 * * @return -1 if group1 should be displayed before group2, 1 otherwise */ - private static int compareGroupsByAttribute(FileGroup group1, FileGroup group2) { + private static int compareGroupsByGroupKey(FileGroup group1, FileGroup group2) { return group1.groupKey.compareTo(group2.groupKey); - } /** * Compare two groups based on the group size. - * Falls back on the attribute if the groups are the same size. + * Falls back on the group key if the groups are the same size. * * @param group1 * @param group2 @@ -148,8 +138,8 @@ class FileGroup implements Comparable { if (group1.files.size() != group2.files.size()) { return -1 * Long.compare(group1.files.size(), group2.files.size()); // High to low } else { - // If the groups have the same size, fall through to the BY_ATTRIBUTE sorting - return compareGroupsByAttribute(group1, group2); + // If the groups have the same size, fall through to the BY_GROUP_KEY sorting + return compareGroupsByGroupKey(group1, group2); } } @@ -157,8 +147,8 @@ class FileGroup implements Comparable { * Enum to specify how to sort the group. */ enum GroupSortingAlgorithm { - BY_GROUP_SIZE, - BY_ATTRIBUTE + BY_GROUP_SIZE, // Sort from largest to smallest group + BY_GROUP_KEY // Sort using the group key (for example, if grouping by size sort from largest to smallest value) } } diff --git a/Core/src/org/sleuthkit/autopsy/newpackage/FileSearch.java b/Core/src/org/sleuthkit/autopsy/newpackage/FileSearch.java index 39b18becbd..2ffcce9f5e 100644 --- a/Core/src/org/sleuthkit/autopsy/newpackage/FileSearch.java +++ b/Core/src/org/sleuthkit/autopsy/newpackage/FileSearch.java @@ -20,14 +20,12 @@ package org.sleuthkit.autopsy.newpackage; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.logging.Level; -import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; @@ -70,10 +68,10 @@ class FileSearch { * @throws FileSearchException */ static LinkedHashMap> runFileSearch( - List filters, + List filters, AttributeType groupAttributeType, FileGroup.GroupSortingAlgorithm groupSortingType, - Comparator fileSortingMethod, + FileSorter.SortingMethod fileSortingMethod, List attributesNeededForGroupingOrSorting, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { @@ -84,11 +82,8 @@ class FileSearch { addAttributes(attributesNeededForGroupingOrSorting, resultFiles, caseDb, centralRepoDb); // Collect everything in the search results - // TODO move add into Searchresults SearchResults searchResults = new SearchResults(groupSortingType, groupAttributeType, fileSortingMethod); - for (ResultFile file : resultFiles) { - searchResults.add(file); - } + searchResults.add(resultFiles); // Return a version of the results in general Java objects return searchResults.toLinkedHashMap(); @@ -112,20 +107,6 @@ class FileSearch { attr.addAttributeToResultFiles(resultFiles, caseDb, centralRepoDb); } } - - /** - * Get a comparator for sorting on file name - * - * @return comparator for case-insensitive sort on the abstract file name field - */ - static Comparator getFileNameComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - return file1.getAbstractFile().getName().compareToIgnoreCase(file2.getAbstractFile().getName()); - } - }; - } /** * Base class for the grouping attributes. @@ -141,13 +122,6 @@ class FileSearch { */ abstract GroupKey getGroupKey(ResultFile file); - /** - * Get the file comparator based on this attribute. - * - * @return the file comparator based on this attribute - */ - abstract Comparator getDefaultFileComparator(); - /** * Add any extra data to the ResultFile object from this attribute. * @@ -216,25 +190,12 @@ class FileSearch { GroupKey getGroupKey(ResultFile file) { return new FileSizeGroupKey(file); } - - @Override - Comparator getDefaultFileComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - // Sort large to small - if (file1.getAbstractFile().getSize() != file2.getAbstractFile().getSize()) { - return -1 * Long.compare(file1.getAbstractFile().getSize(), file2.getAbstractFile().getSize()); - } - - // Secondary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } - }; - } } - static class FileSizeGroupKey extends GroupKey { + /** + * Key representing a file size group + */ + private static class FileSizeGroupKey extends GroupKey { private final FileSize fileSize; FileSizeGroupKey(ResultFile file) { @@ -285,37 +246,12 @@ class FileSearch { GroupKey getGroupKey(ResultFile file) { return new ParentPathGroupKey(file); } - - @Override - Comparator getDefaultFileComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - // Handle missing paths - if (file1.getAbstractFile().getParentPath() == null) { - if (file2.getAbstractFile().getParentPath() == null) { - // Secondary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } else { - return 1; - } - } else if (file2.getAbstractFile().getParentPath() == null) { - return -1; - } - - // Secondary sort on file name if the parent paths are the same - if (file1.getAbstractFile().getParentPath().equals(file2.getAbstractFile().getParentPath())) { - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } - - // Case insensitive comparison on the parent path - return file1.getAbstractFile().getParentPath().compareToIgnoreCase(file2.getAbstractFile().getParentPath()); - } - }; - } } - static class ParentPathGroupKey extends GroupKey { + /** + * Key representing a parent path group + */ + private static class ParentPathGroupKey extends GroupKey { private final String parentPath; ParentPathGroupKey(ResultFile file) { @@ -370,25 +306,12 @@ class FileSearch { GroupKey getGroupKey(ResultFile file) { return new DataSourceGroupKey(file); } - - @Override - Comparator getDefaultFileComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - // Primary sort on data source object ID, small to large - if (file1.getAbstractFile().getDataSourceObjectId() != file2.getAbstractFile().getDataSourceObjectId()) { - return Long.compare(file1.getAbstractFile().getDataSourceObjectId(), file2.getAbstractFile().getDataSourceObjectId()); - } - - // Secondary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } - }; - } } - - static class DataSourceGroupKey extends GroupKey { + + /** + * Key representing a data source group + */ + private static class DataSourceGroupKey extends GroupKey { private final long dataSourceID; private String displayName; @@ -457,42 +380,6 @@ class FileSearch { return new FileTypeGroupKey(file); } - @Override - Comparator getDefaultFileComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - if (file1.getFileType() != file2.getFileType()) { - // Primary sort on the file type enum - return Integer.compare(file1.getFileType().getRanking(), file2.getFileType().getRanking()); - } else { - String mimeType1 = file1.getAbstractFile().getMIMEType(); - String mimeType2 = file2.getAbstractFile().getMIMEType(); - - // Handle missing MIME types - if (mimeType1 == null) { - if (mimeType2 == null) { - // Tertiary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } else { - return 1; - } - } else if (mimeType2 == null) { - return -1; - } - - // Secondary sort on MIME type - if ( ! StringUtils.equals(mimeType1, mimeType2)) { - return mimeType1.compareToIgnoreCase(mimeType2); - } - - // Tertiary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } - } - }; - } - @Override void addAttributeToResultFiles(List files, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { @@ -504,7 +391,10 @@ class FileSearch { } } - static class FileTypeGroupKey extends GroupKey { + /** + * Key representing a file type group + */ + private static class FileTypeGroupKey extends GroupKey { private final FileType fileType; FileTypeGroupKey(ResultFile file) { @@ -556,34 +446,6 @@ class FileSearch { return new KeywordListGroupKey(file); } - @Override - Comparator getDefaultFileComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - - // TODO fix sort - // Force "no keyword hits" to the bottom - if (! file1.hasKeywords()) { - if (! file2.hasKeywords()) { - // Secondary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } - return 1; - }else if (! file2.hasKeywords()) { - return -1; - } - - if (file1.getKeywordListNames().equals(file2.getKeywordListNames())) { - // Secondary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } - return -1; - //return file1.getKeywordListNames().compareToIgnoreCase(file2.getKeywordListNames()); - } - }; - } - @Override void addAttributeToResultFiles(List files, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { @@ -658,8 +520,11 @@ class FileSearch { } } } - - static class KeywordListGroupKey extends GroupKey { + + /** + * Key representing a keyword list group + */ + private static class KeywordListGroupKey extends GroupKey { private final List keywordListNames; private final String keywordListNamesString; @@ -672,11 +537,10 @@ class FileSearch { if (keywordListNames.isEmpty()) { keywordListNamesString = Bundle.FileSearch_KeywordListGroupKey_noKeywords(); } else { - keywordListNamesString = String.join(",", keywordListNames); + keywordListNamesString = String.join(",", keywordListNames); // NON-NLS } } - @Override String getDisplayName() { return keywordListNamesString; @@ -686,6 +550,18 @@ class FileSearch { public int compareTo(GroupKey otherGroupKey) { if (otherGroupKey instanceof KeywordListGroupKey) { KeywordListGroupKey otherKeywordListNamesGroupKey = (KeywordListGroupKey)otherGroupKey; + + // Put the empty list at the end + if (keywordListNames.isEmpty()) { + if (otherKeywordListNamesGroupKey.keywordListNames.isEmpty()) { + return 0; + } else { + return 1; + } + } else if (otherKeywordListNamesGroupKey.keywordListNames.isEmpty()) { + return -1; + } + return keywordListNamesString.compareTo(otherKeywordListNamesGroupKey.keywordListNamesString); } else { return compareClassNames(otherGroupKey); @@ -701,7 +577,7 @@ class FileSearch { if (!(otherKey instanceof KeywordListGroupKey)) { return false; } - // TODO put no kw group last + KeywordListGroupKey otherKeywordListGroupKey = (KeywordListGroupKey)otherKey; return keywordListNamesString.equals(otherKeywordListGroupKey.keywordListNamesString); } @@ -722,21 +598,6 @@ class FileSearch { return new FrequencyGroupKey(file); } - @Override - Comparator getDefaultFileComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - if (file1.getFrequency() != file2.getFrequency()) { - return Long.compare(file1.getFrequency().getRanking(), file2.getFrequency().getRanking()); - } - - // Secondary sort on file name - return file1.getAbstractFile().getName().compareToIgnoreCase(file1.getAbstractFile().getName()); - } - }; - } - @Override void addAttributeToResultFiles(List files, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { @@ -763,7 +624,10 @@ class FileSearch { } } - static class FrequencyGroupKey extends GroupKey { + /** + * Key representing a central repository frequency group + */ + private static class FrequencyGroupKey extends GroupKey { private final Frequency frequency; FrequencyGroupKey(ResultFile file) { @@ -814,15 +678,12 @@ class FileSearch { GroupKey getGroupKey(ResultFile file) { return new NoGroupingGroupKey(); } - - @Override - Comparator getDefaultFileComparator() { - // Default to sort by file name - return FileSearch.getFileNameComparator(); - } } - static class NoGroupingGroupKey extends GroupKey { + /** + * Dummy key for when there is no grouping. All files will have the same key. + */ + private static class NoGroupingGroupKey extends GroupKey { NoGroupingGroupKey() { // Nothing to save - all files will get the same GroupKey @@ -838,6 +699,7 @@ class FileSearch { @Override public int compareTo(GroupKey otherGroupKey) { + // As long as the other key is the same type, they are equal if (otherGroupKey instanceof NoGroupingGroupKey) { return 0; } else { @@ -855,6 +717,7 @@ class FileSearch { return false; } + // As long as the other key is the same type, they are equal return true; } diff --git a/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchFiltering.java b/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchFiltering.java index 330752266e..63c17bcf28 100644 --- a/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchFiltering.java +++ b/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchFiltering.java @@ -33,13 +33,17 @@ import org.sleuthkit.datamodel.TskCoreException; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; /** * Run various filters to return a subset of files from the current case. */ class FileSearchFiltering { + private final static Logger logger = Logger.getLogger(FileSearchFiltering.class.getName()); + /** * Run the given filters to get a list of matching files. * @@ -48,39 +52,37 @@ class FileSearchFiltering { * @param crDb The central repo. Can be null as long as no filters need it. * @return */ - static List runQueries(List filters, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { + static List runQueries(List filters, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { if (caseDb == null) { throw new FileSearchException("Case DB parameter is null"); // NON-NLS } - // Debug - print out the current filters. Could perhaps be an info statement. - System.out.println("Running filters: "); - for (SubFilter filter : filters) { - System.out.println(" " + filter.getDesc()); + // Record the selected filters + String filterStr = ""; + for (FileFilter filter : filters) { + filterStr += " " + filter.getDesc() + "\n"; } - System.out.println("\n"); + logger.log(Level.INFO, "Running filters:\n{0}", filterStr); // Combine all the SQL queries from the filters into one query // TODO - maybe exclude directories and other non-file objects? String combinedQuery = ""; - for (SubFilter subFilter : filters) { - if ( ! subFilter.getSQL().isEmpty()) { + for (FileFilter filter : filters) { + if ( ! filter.getWhereClause().isEmpty()) { if ( ! combinedQuery.isEmpty()) { combinedQuery += " AND "; // NON-NLS } - combinedQuery += "(" + subFilter.getSQL() + ")"; // NON-NLS + combinedQuery += "(" + filter.getWhereClause() + ")"; // NON-NLS } } try { // Get all matching abstract files List resultList = new ArrayList<>(); - - // Debug - print the SQL. could also be info - System.out.println("SQL query: " + combinedQuery + "\n"); - + if ( ! combinedQuery.isEmpty()) { + logger.log(Level.INFO, "Running SQL query: {0}", combinedQuery); List sqlResults = caseDb.findAllFilesWhere(combinedQuery); // If there are no results, return now @@ -95,11 +97,12 @@ class FileSearchFiltering { } // Now run any non-SQL filters. Note that resultList could be empty at this point if we had no SQL queries - - // getMatchingFiles() will interpret this as no filters have been run up to this point + // applyAlternateFilter() will interpret this as no filters have been run up to this point // and act accordingly. - for (SubFilter subFilter : filters) { - if (subFilter.useAlternameFilter()) { // TODO typo - resultList = subFilter.getMatchingFiles(resultList, caseDb, centralRepoDb); // TODO rename + // TODO maybe it is an error to have no SQL query + for (FileFilter subFilter : filters) { + if (subFilter.useAlternateFilter()) { + resultList = subFilter.applyAlternateFilter(resultList, caseDb, centralRepoDb); } // There are no matches for the filters run so far, so return @@ -117,18 +120,18 @@ class FileSearchFiltering { /** * Base class for the filters. */ - static abstract class SubFilter { + static abstract class FileFilter { /** * Returns part of a query on the tsk_files table that can be AND-ed with other pieces * @return the SQL query or an empty string if there is no SQL query for this filter. */ - abstract String getSQL(); + abstract String getWhereClause(); /** - * Indicates whether this filter needs to use the secondary, non-SQL method getMatchingFiles(). + * Indicates whether this filter needs to use the secondary, non-SQL method applyAlternateFilter(). * @return false by default */ - boolean useAlternameFilter() { + boolean useAlternateFilter() { return false; } @@ -145,7 +148,7 @@ class FileSearchFiltering { * * @throws FileSearchException */ - List getMatchingFiles (List currentResults, SleuthkitCase caseDb, + List applyAlternateFilter (List currentResults, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { return new ArrayList<>(); } @@ -161,7 +164,7 @@ class FileSearchFiltering { /** * A filter for specifying the file size */ - static class SizeSubFilter extends SubFilter { + static class SizeSubFilter extends FileFilter { private final List fileSizes; /** @@ -174,7 +177,7 @@ class FileSearchFiltering { } @Override - String getSQL() { + String getWhereClause() { String queryStr = ""; // NON-NLS for (FileSize size : fileSizes) { if (! queryStr.isEmpty()) { @@ -249,7 +252,7 @@ class FileSearchFiltering { /** * A filter for specifying parent path (either full path or substring) */ - static class ParentSubFilter extends SubFilter { + static class ParentSubFilter extends FileFilter { private final List parentSearchTerms; /** @@ -262,7 +265,7 @@ class FileSearchFiltering { } @Override - String getSQL() { + String getWhereClause() { String queryStr = ""; // NON-NLS for (ParentSearchTerm searchTerm : parentSearchTerms) { if (! queryStr.isEmpty()) { @@ -301,7 +304,7 @@ class FileSearchFiltering { /** * A filter for specifying data sources */ - static class DataSourceSubFilter extends SubFilter { + static class DataSourceSubFilter extends FileFilter { private final List dataSources; /** @@ -314,7 +317,7 @@ class FileSearchFiltering { } @Override - String getSQL() { + String getWhereClause() { String queryStr = ""; // NON-NLS for (DataSource ds : dataSources) { if (! queryStr.isEmpty()) { @@ -352,7 +355,7 @@ class FileSearchFiltering { * A filter for specifying keyword list names. * A file must contain a keyword from one of the given lists to pass. */ - static class KeywordListSubFilter extends SubFilter { + static class KeywordListSubFilter extends FileFilter { private final List listNames; /** @@ -364,7 +367,7 @@ class FileSearchFiltering { } @Override - String getSQL() { + String getWhereClause() { String keywordListPart = ""; // NON-NLS for (String listName : listNames) { if (! keywordListPart.isEmpty()) { @@ -402,7 +405,7 @@ class FileSearchFiltering { /** * A filter for specifying file types. */ - static class FileTypeSubFilter extends SubFilter { + static class FileTypeSubFilter extends FileFilter { private final List categories; /** @@ -414,7 +417,7 @@ class FileSearchFiltering { } @Override - String getSQL() { + String getWhereClause() { String queryStr = ""; // NON-NLS for (FileTypeUtils.FileTypeCategory cat : categories) { for (String type : cat.getMediaTypes()) { @@ -450,7 +453,7 @@ class FileSearchFiltering { /** * A filter for specifying frequency in the central repository. */ - static class FrequencySubFilter extends SubFilter { + static class FrequencySubFilter extends FileFilter { private final List frequencies; @@ -464,19 +467,19 @@ class FileSearchFiltering { } @Override - String getSQL() { + String getWhereClause() { // Since this relies on the central repository database, there is no // query on the case database. return ""; // NON-NLS } @Override - boolean useAlternameFilter() { + boolean useAlternateFilter() { return true; } @Override - List getMatchingFiles (List currentResults, SleuthkitCase caseDb, + List applyAlternateFilter (List currentResults, SleuthkitCase caseDb, EamDb centralRepoDb) throws FileSearchException { if (centralRepoDb == null) { diff --git a/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchTestAction.java b/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchTestAction.java index 9f437b6259..1de8ff3ec3 100644 --- a/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchTestAction.java +++ b/Core/src/org/sleuthkit/autopsy/newpackage/FileSearchTestAction.java @@ -20,32 +20,15 @@ package org.sleuthkit.autopsy.newpackage; import java.util.*; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.InvalidPathException; -import java.util.logging.Level; -import java.beans.PropertyChangeListener; -import java.beans.PropertyChangeEvent; -import javax.swing.JOptionPane; -import java.awt.Frame; -import javax.swing.SwingWorker; -import org.apache.commons.io.FileUtils; import org.openide.awt.ActionID; import org.openide.awt.ActionReference; import org.openide.awt.ActionRegistration; import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; -import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.core.UserPreferences; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.datamodel.utils.FileTypeUtils; import org.sleuthkit.autopsy.progress.ModalDialogProgressIndicator; -import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.DataSource; @ActionID(category = "Tools", id = "org.sleuthkit.autopsy.newpackage.FileSearchTestAction") @@ -68,27 +51,27 @@ public final class FileSearchTestAction extends CallableSystemAction { System.out.println("\n#########################\nTesting file search!!!"); // Set up all the test filters - FileSearchFiltering.SubFilter size_medSmallXS = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.MEDIUM, FileSearchData.FileSize.SMALL, FileSearchData.FileSize.XS)); - FileSearchFiltering.SubFilter size_XL = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.XL)); - FileSearchFiltering.SubFilter size_largeXL = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.LARGE, FileSearchData.FileSize.XL)); - FileSearchFiltering.SubFilter size_medLargeXL = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.MEDIUM, FileSearchData.FileSize.LARGE, FileSearchData.FileSize.XL)); + FileSearchFiltering.FileFilter size_medSmallXS = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.MEDIUM, FileSearchData.FileSize.SMALL, FileSearchData.FileSize.XS)); + FileSearchFiltering.FileFilter size_XL = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.XL)); + FileSearchFiltering.FileFilter size_largeXL = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.LARGE, FileSearchData.FileSize.XL)); + FileSearchFiltering.FileFilter size_medLargeXL = new FileSearchFiltering.SizeSubFilter(Arrays.asList(FileSearchData.FileSize.MEDIUM, FileSearchData.FileSize.LARGE, FileSearchData.FileSize.XL)); - FileSearchFiltering.SubFilter kw_alphaBeta = new FileSearchFiltering.KeywordListSubFilter(Arrays.asList("Alpha", "Beta")); - FileSearchFiltering.SubFilter kw_alpha = new FileSearchFiltering.KeywordListSubFilter(Arrays.asList("Alpha")); + FileSearchFiltering.FileFilter kw_alphaBeta = new FileSearchFiltering.KeywordListSubFilter(Arrays.asList("Alpha", "Beta")); + FileSearchFiltering.FileFilter kw_alpha = new FileSearchFiltering.KeywordListSubFilter(Arrays.asList("Alpha")); - FileSearchFiltering.SubFilter freq_uniqueRare = new FileSearchFiltering.FrequencySubFilter(Arrays.asList(FileSearchData.Frequency.UNIQUE, FileSearchData.Frequency.RARE)); + FileSearchFiltering.FileFilter freq_uniqueRare = new FileSearchFiltering.FrequencySubFilter(Arrays.asList(FileSearchData.Frequency.UNIQUE, FileSearchData.Frequency.RARE)); - FileSearchFiltering.SubFilter path_II = new FileSearchFiltering.ParentSubFilter(Arrays.asList(new FileSearchFiltering.ParentSearchTerm("II", false))); - FileSearchFiltering.SubFilter path_IIfolderA = new FileSearchFiltering.ParentSubFilter(Arrays.asList(new FileSearchFiltering.ParentSearchTerm("II", false), + FileSearchFiltering.FileFilter path_II = new FileSearchFiltering.ParentSubFilter(Arrays.asList(new FileSearchFiltering.ParentSearchTerm("II", false))); + FileSearchFiltering.FileFilter path_IIfolderA = new FileSearchFiltering.ParentSubFilter(Arrays.asList(new FileSearchFiltering.ParentSearchTerm("II", false), new FileSearchFiltering.ParentSearchTerm("/Rare in CR/Folder A/", true))); - FileSearchFiltering.SubFilter type_video = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.VIDEO)); - FileSearchFiltering.SubFilter type_imageAudio = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.IMAGE, + FileSearchFiltering.FileFilter type_video = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.VIDEO)); + FileSearchFiltering.FileFilter type_imageAudio = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.IMAGE, FileTypeUtils.FileTypeCategory.AUDIO)); - FileSearchFiltering.SubFilter type_image = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.IMAGE)); - FileSearchFiltering.SubFilter type_doc = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.DOCUMENTS)); + FileSearchFiltering.FileFilter type_image = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.IMAGE)); + FileSearchFiltering.FileFilter type_doc = new FileSearchFiltering.FileTypeSubFilter(Arrays.asList(FileTypeUtils.FileTypeCategory.DOCUMENTS)); - FileSearchFiltering.SubFilter ds_46 = null; + FileSearchFiltering.FileFilter ds_46 = null; try { DataSource ds = Case.getCurrentCase().getSleuthkitCase().getDataSource(46); ds_46 = new FileSearchFiltering.DataSourceSubFilter(Arrays.asList(ds)); @@ -97,14 +80,15 @@ public final class FileSearchTestAction extends CallableSystemAction { return; } + ////////////////////// // Set up test // Select test filters - List filters = new ArrayList<>(); + List filters = new ArrayList<>(); filters.add(size_medSmallXS); //filters.add(kw_alpha); - filters.add(freq_uniqueRare); + //filters.add(freq_uniqueRare); // Choose grouping attribute //FileSearch.AttributeType groupingAttr = new FileSearch.FileTypeAttribute(); @@ -112,17 +96,14 @@ public final class FileSearchTestAction extends CallableSystemAction { FileSearch.AttributeType groupingAttr = new FileSearch.KeywordListAttribute(); //FileSearch.AttributeType groupingAttr = new FileSearch.FrequencyAttribute(); //FileSearch.AttributeType groupingAttr = new FileSearch.ParentPathAttribute(); + //FileSearch.AttributeType groupingAttr = new FileSearch.NoGroupingAttribute(); // Choose group sorting method - //FileGroup.GroupSortingAlgorithm groupSortAlgorithm = FileGroup.GroupSortingAlgorithm.BY_ATTRIBUTE; - FileGroup.GroupSortingAlgorithm groupSortAlgorithm = FileGroup.GroupSortingAlgorithm.BY_GROUP_SIZE; + FileGroup.GroupSortingAlgorithm groupSortAlgorithm = FileGroup.GroupSortingAlgorithm.BY_GROUP_KEY; + //FileGroup.GroupSortingAlgorithm groupSortAlgorithm = FileGroup.GroupSortingAlgorithm.BY_GROUP_SIZE; // Choose file sorting method - Comparator fileSort = FileSearch.getFileNameComparator(); - //Comparator fileSort = new FileSearch.FrequencyAttribute().getDefaultFileComparator(); - //Comparator fileSort = new FileSearch.FileSizeAttribute().getDefaultFileComparator(); - //Comparator fileSort = new FileSearch.ParentPathAttribute().getDefaultFileComparator(); - //Comparator fileSort = new FileSearch.FileTypeAttribute().getDefaultFileComparator(); + FileSorter.SortingMethod fileSort = FileSorter.SortingMethod.BY_FILE_SIZE; EamDb crDb = null; if (EamDb.isEnabled()) { diff --git a/Core/src/org/sleuthkit/autopsy/newpackage/FileSorter.java b/Core/src/org/sleuthkit/autopsy/newpackage/FileSorter.java index e8a2504921..6c67a8ad4f 100644 --- a/Core/src/org/sleuthkit/autopsy/newpackage/FileSorter.java +++ b/Core/src/org/sleuthkit/autopsy/newpackage/FileSorter.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.newpackage; @@ -10,15 +23,22 @@ import java.util.Comparator; import java.util.List; /** - * + * Class used to sort ResultFiles using the supplied method. */ class FileSorter implements Comparator { - List> comparators = new ArrayList<>(); + private final List> comparators = new ArrayList<>(); + /** + * Set up the sorter using the supplied sorting method. + * The sorting is defined by a list of ResultFile comparators. These + * comparators will be run in order until one returns a non-zero result. + * + * @param method The method that should be used to sort the files + */ FileSorter(SortingMethod method) { - // Set up the comparators that should run on the files + // Set up the primary comparators that should applied to the files switch (method) { case BY_DATA_SOURCE: comparators.add(getDataSourceComparator()); @@ -34,12 +54,16 @@ class FileSorter implements Comparator { comparators.add(getFrequencyComparator()); break; case BY_KEYWORD_LIST_NAMES: + comparators.add(getKeywordListNameComparator()); break; case BY_PARENT_PATH: + comparators.add(getParentPathComparator()); break; case BY_FILE_NAME: comparators.add(getFileNameComparator()); + break; default: + // The default comparator will be added afterward break; } @@ -51,73 +75,106 @@ class FileSorter implements Comparator { @Override public int compare(ResultFile file1, ResultFile file2) { - return 0; + int result = 0; + for (Comparator comp : comparators) { + result = comp.compare(file1, file2); + if (result != 0) { + return result; + } + } + + // The files are the same + return result; } + /** + * Compare files using data source ID. Will order smallest to largest. + * + * @return -1 if file1 has the lower data source ID, 0 if equal, 1 otherwise + */ private static Comparator getDataSourceComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - // Sort large to small - return Long.compare(file1.getAbstractFile().getDataSourceObjectId(), file2.getAbstractFile().getDataSourceObjectId()); - } - }; + return (ResultFile file1, ResultFile file2) -> Long.compare(file1.getAbstractFile().getDataSourceObjectId(), file2.getAbstractFile().getDataSourceObjectId()); } + /** + * Compare files using their FileType enum. Orders based on the ranking + * in the FileType enum. + * + * @return -1 if file1 has the lower FileType value, 0 if equal, 1 otherwise + */ private static Comparator getFileTypeComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - return Integer.compare(file1.getFileType().getRanking(), file2.getFileType().getRanking()); - } - }; + return (ResultFile file1, ResultFile file2) -> Integer.compare(file1.getFileType().getRanking(), file2.getFileType().getRanking()); } + /** + * Compare files using a concatenated version of keyword list names. Alphabetical by + * the list names with files with no keyword list hits going last. + * + * @return -1 if file1 has the earliest combined keyword list name, 0 if equal, 1 otherwise + */ + private static Comparator getKeywordListNameComparator() { + return (ResultFile file1, ResultFile file2) -> { + // Put empty lists at the bottom + if (file1.getKeywordListNames().isEmpty()) { + if (file2.getKeywordListNames().isEmpty()) { + return 0; + } + return 1; + } else if (file2.getKeywordListNames().isEmpty()) { + return -1; + } + + String list1 = String.join(",", file1.getKeywordListNames()); + String list2 = String.join(",", file2.getKeywordListNames()); + return compareStrings(list1, list2); + }; + } + + /** + * Compare files based on parent path. Order alphabetically. + * + * @return -1 if file1's path comes first alphabetically, 0 if equal, 1 otherwise + */ private static Comparator getParentPathComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - return compareStrings(file1.getAbstractFile().getParentPath(), file2.getAbstractFile().getParentPath()); - } - }; + return (ResultFile file1, ResultFile file2) -> compareStrings(file1.getAbstractFile().getParentPath(), file2.getAbstractFile().getParentPath()); } + /** + * Compare files based on number of occurrences in the central repository. + * Order from most rare to least rare Frequency enum. + * + * @return -1 if file1's rarity is lower than file2, 0 if equal, 1 otherwise + */ private static Comparator getFrequencyComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - return Integer.compare(file1.getFrequency().getRanking(), file2.getFrequency().getRanking()); - } - }; + return (ResultFile file1, ResultFile file2) -> Integer.compare(file1.getFrequency().getRanking(), file2.getFrequency().getRanking()); } + /** + * Compare files based on MIME type. Order is alphabetical. + * + * @return -1 if file1's MIME type comes before file2's, 0 if equal, 1 otherwise + */ private static Comparator getMIMETypeComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - // Secondary sort on the MIME type - return compareStrings(file1.getAbstractFile().getMIMEType(), file2.getAbstractFile().getMIMEType()); - } - }; + return (ResultFile file1, ResultFile file2) -> compareStrings(file1.getAbstractFile().getMIMEType(), file2.getAbstractFile().getMIMEType()); } + /** + * Compare files based on size. Order large to small. + * + * @return -1 if file1 is larger than file2, 0 if equal, 1 otherwise + */ private static Comparator getFileSizeComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - // Sort large to small - return -1 * Long.compare(file1.getAbstractFile().getSize(), file2.getAbstractFile().getSize()); - } - }; + return (ResultFile file1, ResultFile file2) -> -1 * Long.compare(file1.getAbstractFile().getSize(), file2.getAbstractFile().getSize()) // Sort large to small + ; } + /** + * Compare files based on file name. Order alphabetically. + * + * @return -1 if file1 comes before file2, 0 if equal, 1 otherwise + */ private static Comparator getFileNameComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - return compareStrings(file1.getAbstractFile().getName(), file2.getAbstractFile().getName()); - } - }; + return (ResultFile file1, ResultFile file2) -> compareStrings(file1.getAbstractFile().getName(), file2.getAbstractFile().getName()); } /** @@ -126,19 +183,16 @@ class FileSorter implements Comparator { * should always include something like the object ID to ensure a * consistent sorting when the rest of the compared fields are the same. * - * @return + * @return -1 if file1 comes before file2, 0 if equal, 1 otherwise */ private static Comparator getDefaultComparator() { - return new Comparator() { - @Override - public int compare(ResultFile file1, ResultFile file2) { - // For now, compare file names and then object ID (to ensure a consistent sort) - int result = getFileNameComparator().compare(file1, file2); - if (result == 0) { - return Long.compare(file1.getAbstractFile().getId(), file2.getAbstractFile().getId()); - } - return result; + return (ResultFile file1, ResultFile file2) -> { + // Compare file names and then object ID (to ensure a consistent sort) + int result = getFileNameComparator().compare(file1, file2); + if (result == 0) { + return Long.compare(file1.getAbstractFile().getId(), file2.getAbstractFile().getId()); } + return result; }; } @@ -148,7 +202,7 @@ class FileSorter implements Comparator { * @param s1 * @param s2 * - * @return + * @return -1 if s1 comes before s2, 0 if equal, 1 otherwise */ private static int compareStrings(String s1, String s2) { if (s1 == null) { @@ -160,13 +214,16 @@ class FileSorter implements Comparator { return s1.compareTo(s2); } + /** + * Enum for selecting the primary method for sorting result files. + */ enum SortingMethod { - BY_DATA_SOURCE, - BY_FILE_NAME, - BY_FILE_SIZE, - BY_FILE_TYPE, - BY_FREQUENCY, - BY_KEYWORD_LIST_NAMES, - BY_PARENT_PATH; + BY_DATA_SOURCE, // Sort in increasing order of data source ID + BY_FILE_NAME, // Sort alphabetically by file name + BY_FILE_SIZE, // Sort in decreasing order of size + BY_FILE_TYPE, // Sort in order of file type (defined in FileType enum), with secondary sort on MIME type + BY_FREQUENCY, // Sort by decreasing rarity in the central repository + BY_KEYWORD_LIST_NAMES, // Sort alphabetically by list of keyword list names found + BY_PARENT_PATH; // Sort alphabetically by path } } diff --git a/Core/src/org/sleuthkit/autopsy/newpackage/ResultFile.java b/Core/src/org/sleuthkit/autopsy/newpackage/ResultFile.java index 3ee3243c0d..0885d55c86 100644 --- a/Core/src/org/sleuthkit/autopsy/newpackage/ResultFile.java +++ b/Core/src/org/sleuthkit/autopsy/newpackage/ResultFile.java @@ -88,7 +88,7 @@ class ResultFile { keywordListNames.add(keywordListName); } - // Sort the list so the getKeywordListNames () will be consistent regardless of the order added + // Sort the list so the getKeywordListNames() will be consistent regardless of the order added Collections.sort(keywordListNames); } @@ -101,15 +101,6 @@ class ResultFile { return keywordListNames; } - /** - * Check whether this file has any keyword list matches - * - * @return true if it has a keyword list match, false otherwise - */ - boolean hasKeywords() { - return ! keywordListNames.isEmpty(); - } - /** * Get the AbstractFile * diff --git a/Core/src/org/sleuthkit/autopsy/newpackage/SearchResults.java b/Core/src/org/sleuthkit/autopsy/newpackage/SearchResults.java index 4c25170a61..a853d54e00 100644 --- a/Core/src/org/sleuthkit/autopsy/newpackage/SearchResults.java +++ b/Core/src/org/sleuthkit/autopsy/newpackage/SearchResults.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.newpackage; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -34,9 +33,9 @@ class SearchResults { private final FileGroup.GroupSortingAlgorithm groupSortingType; private final FileSearch.AttributeType attrType; - private final Comparator fileSortingMethod; + private final FileSorter fileSorter; - private final Map groupMap = new HashMap<>(); + private final Map groupMap = new HashMap<>(); private List groupList = null; /** @@ -47,24 +46,27 @@ class SearchResults { * @param fileSortingMethod The method that should be used to sortGroupsAndFiles the files in each group */ SearchResults(FileGroup.GroupSortingAlgorithm groupSortingType, FileSearch.AttributeType attrType, - Comparator fileSortingMethod) { + FileSorter.SortingMethod fileSortingMethod) { this.groupSortingType = groupSortingType; this.attrType = attrType; - this.fileSortingMethod = fileSortingMethod; + this.fileSorter = new FileSorter(fileSortingMethod); } /** - * Add a ResultFile to the results + * Add a list of ResultFile to the results * - * @param file the ResultFile + * @param files the ResultFiles */ - void add(ResultFile file) { - FileSearch.GroupKey groupKey = attrType.getGroupKey(file); - - if ( ! groupMap.containsKey(groupKey)) { - groupMap.put(groupKey, new FileGroup(groupSortingType, fileSortingMethod, groupKey)); + void add(List files) { + for (ResultFile file : files) { + // Add the file to the appropriate group, creating it if necessary + FileSearch.GroupKey groupKey = attrType.getGroupKey(file); + + if ( ! groupMap.containsKey(groupKey)) { + groupMap.put(groupKey, new FileGroup(groupSortingType, groupKey)); + } + groupMap.get(groupKey).addFile(file); } - groupMap.get(groupKey).addFile(file); } /** @@ -74,7 +76,7 @@ class SearchResults { // First sortGroupsAndFiles the files for (FileGroup group : groupMap.values()) { - group.sortFiles(); + group.sortFiles(fileSorter); } // Now put the groups in a list and sortGroupsAndFiles them @@ -82,6 +84,7 @@ class SearchResults { Collections.sort(groupList); // Debugging - print the results here + // This code should remain until we have a working UI. System.out.println("\nSearchResults"); for (FileGroup group : groupList) { System.out.println(" " + group.getDisplayName());