From 7754011b69f62ea20a49ff74b13696fa2d999aa6 Mon Sep 17 00:00:00 2001 From: Andrew Ziehl Date: Tue, 17 Apr 2018 16:46:44 -0700 Subject: [PATCH] Update UI element variable names. Add filtering by mime_type based on check box selection. Needs Brian's changes to use executeQuery instead. --- .../commonfilesearch/AllDataSources.java | 10 +- .../commonfilesearch/Bundle.properties | 6 +- .../CommonFilesMetaDataBuilder.java | 77 ++++++++++++- .../commonfilesearch/CommonFilesPanel.form | 91 +++++++++------ .../commonfilesearch/CommonFilesPanel.java | 105 +++++++++++------- .../commonfilesearch/SingleDataSource.java | 10 +- 6 files changed, 207 insertions(+), 92 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSources.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSources.java index ebf3c1bafc..5138b51bd7 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSources.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllDataSources.java @@ -26,14 +26,16 @@ import java.util.Map; */ class AllDataSources extends CommonFilesMetaDataBuilder { - private static final String WHERE_CLAUSE = "md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL) GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; + private static final String WHERE_CLAUSE = "md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL)%s GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; - public AllDataSources(Map dataSourceIdMap) { - super(dataSourceIdMap); + public AllDataSources(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) { + super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType); } @Override protected String buildSqlWhereClause() { - return AllDataSources.WHERE_CLAUSE; + + Object[] args = new String[]{determineMimeTypeFilter()}; + return String.format(AllDataSources.WHERE_CLAUSE, args); } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/Bundle.properties b/Core/src/org/sleuthkit/autopsy/commonfilesearch/Bundle.properties index 55facc0910..270b14bf98 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/Bundle.properties @@ -1,12 +1,12 @@ CommonFilesPanel.searchButton.text=Search CommonFilesPanel.withinDataSourceRadioButton.text=Files must be in data source: CommonFilesPanel.allDataSourcesRadioButton.text=Files can be in any data source -CommonFilesPanel.jLabel1.text=Search for files that exist in multiple data sources: CommonFilesPanel.cancelButton.text=Cancel CommonFilesPanel.cancelButton.actionCommand=Cancel CommonFilesPanel.allRadioButton.text=Match on all file types CommonFilesPanel.selectedFileCategoriesButton.text=Match on the following file categories: -CommonFilesPanel.jCheckBox1.text=Pictures and Videos -CommonFilesPanel.jCheckBox2.text=Documents CommonFilesPanel.selectedFileCategoriesButton.toolTipText=Select from the options below... CommonFilesPanel.allRadioButton.toolTipText=No filtering applied to results... +CommonFilesPanel.pictureVideoCheckbox.text=Pictures and Videos +CommonFilesPanel.documentsCheckbox.text=Documents +CommonFilesPanel.commonFilesSearchLabel.text=Search for files that exist in multiple data sources: diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java index 01711fbfae..b72d110601 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesMetaDataBuilder.java @@ -21,14 +21,18 @@ package org.sleuthkit.autopsy.commonfilesearch; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.SleuthkitCase; @@ -44,9 +48,57 @@ import org.sleuthkit.datamodel.TskCoreException; abstract class CommonFilesMetaDataBuilder { private final Map dataSourceIdToNameMap; + private final boolean filterByMedia; + private final boolean filterByDoc; + private final String filterByMimeTypesWhereClause = " and mime_type in (%s)"; // where %s is csv list of mime_types to filter on - CommonFilesMetaDataBuilder(Map dataSourceIdMap) { + /* + * The set of the MIME types that will be checked for extension mismatches + * when checkType is ONLY_MEDIA. + */ + private static final Set MEDIA_PICS_VIDEO_MIME_TYPES = Stream.of( + "image/bmp", + "image/gif", + "image/jpeg", + "image/png", + "image/tiff", + "image/x-ms-bmp", + "image/x-icon", + "video/webm", + "video/3gpp", + "video/3gpp2", + "video/ogg", + "video/mpeg", + "video/x-msvideo" + ).collect(Collectors.toSet()); + + /* + * The set of the MIME types that will be checked for extension mismatches + * when checkType is ONLY_TEXT_FILES. + */ + private static final Set TEXT_FILES_MIME_TYPES = Stream.of( + "text/plain", + "application/rtf", + "application/pdf", + "text/css", + "text/html", + "text/csv", + "application/json", + "application/javascript", + "application/xml", + "text/calendar", + "application/msword", //NON-NLS + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", //NON-NLS + "application/vnd.ms-powerpoint", //NON-NLS + "application/vnd.openxmlformats-officedocument.presentationml.presentation", //NON-NLS + "application/vnd.ms-excel", //NON-NLS + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" //NON-NLS + ).collect(Collectors.toSet()); + + CommonFilesMetaDataBuilder(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) { dataSourceIdToNameMap = dataSourceIdMap; + filterByMedia = filterByMediaMimeType; + filterByDoc = filterByDocMimeType; } private void addDataSource(Set dataSources, AbstractFile file, Map dataSourceIdToNameMap) { @@ -95,8 +147,8 @@ abstract class CommonFilesMetaDataBuilder { for (AbstractFile file : files) { String currentMd5 = file.getMd5Hash(); - if((currentMd5 == null) || (HashUtility.isNoDataMd5(currentMd5))) { - continue; + if ((currentMd5 == null) || (HashUtility.isNoDataMd5(currentMd5))) { + continue; } if (parentNodes.containsKey(currentMd5)) { parentNodes.get(currentMd5).add(file); @@ -122,4 +174,23 @@ abstract class CommonFilesMetaDataBuilder { List files = sleuthkitCase.findAllFilesWhere(whereClause); return files; } + + String determineMimeTypeFilter() { + StringBuilder mimeTypeFilter = new StringBuilder(); + Set mimeTypesToFilterOn = new HashSet<>(); + String mimeTypeString = ""; + if(filterByMedia) { + mimeTypesToFilterOn.addAll(MEDIA_PICS_VIDEO_MIME_TYPES); + } + if(filterByDoc) { + mimeTypesToFilterOn.addAll(TEXT_FILES_MIME_TYPES); + } + if(mimeTypesToFilterOn.size() > 0) { + for (String mimeType : mimeTypesToFilterOn) { + mimeTypeFilter.append("\"" + mimeType + "\","); + } + mimeTypeString = mimeTypeFilter.toString().substring(0, mimeTypeFilter.length() - 1); + } + return String.format(filterByMimeTypesWhereClause, new Object[]{mimeTypeString}); + } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.form b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.form index d64b0ee96f..a005b6d8c8 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.form @@ -27,35 +27,46 @@ - + - - - - - - + + + + - - - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + - + - @@ -63,25 +74,33 @@ - + - + - - - - - - - - - - + + + + + + + + + + + + + + + + + + @@ -143,10 +162,10 @@ - + - + @@ -192,19 +211,19 @@ - + - + - + - + diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java index 5020ab894e..112f6c226c 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java @@ -246,13 +246,22 @@ public final class CommonFilesPanel extends javax.swing.JPanel { Long dataSourceId = determineDataSourceId(); CommonFilesMetaDataBuilder builder; - + boolean filterByMedia = false; + boolean filterByDocuments = false; + if(selectedFileCategoriesButton.isSelected()) { + if(pictureVideoCheckbox.isSelected()) { + filterByMedia = true; + } + if(documentsCheckbox.isSelected()) { + filterByDocuments = true; + } + } if (dataSourceId == CommonFilesPanel.NO_DATA_SOURCE_SELECTED) { - builder = new AllDataSources(dataSourceMap); + builder = new AllDataSources(dataSourceMap, filterByMedia, filterByDocuments); setTitleForAllDataSources(); } else { - builder = new SingleDataSource(dataSourceId, dataSourceMap); + builder = new SingleDataSource(dataSourceId, dataSourceMap, filterByMedia, filterByDocuments); setTitleForSingleSource(dataSourceId); } @@ -377,12 +386,12 @@ public final class CommonFilesPanel extends javax.swing.JPanel { allDataSourcesRadioButton = new javax.swing.JRadioButton(); withinDataSourceRadioButton = new javax.swing.JRadioButton(); selectDataSourceComboBox = new javax.swing.JComboBox<>(); - jLabel1 = new javax.swing.JLabel(); + commonFilesSearchLabel = new javax.swing.JLabel(); cancelButton = new javax.swing.JButton(); allRadioButton = new javax.swing.JRadioButton(); selectedFileCategoriesButton = new javax.swing.JRadioButton(); - jCheckBox1 = new javax.swing.JCheckBox(); - jCheckBox2 = new javax.swing.JCheckBox(); + pictureVideoCheckbox = new javax.swing.JCheckBox(); + documentsCheckbox = new javax.swing.JCheckBox(); setPreferredSize(new java.awt.Dimension(300, 200)); @@ -422,8 +431,8 @@ public final class CommonFilesPanel extends javax.swing.JPanel { } }); - org.openide.awt.Mnemonics.setLocalizedText(jLabel1, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.jLabel1.text")); // NOI18N - jLabel1.setFocusable(false); + org.openide.awt.Mnemonics.setLocalizedText(commonFilesSearchLabel, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.commonFilesSearchLabel.text")); // NOI18N + commonFilesSearchLabel.setFocusable(false); org.openide.awt.Mnemonics.setLocalizedText(cancelButton, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.cancelButton.text")); // NOI18N cancelButton.setActionCommand(org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.cancelButton.actionCommand")); // NOI18N @@ -443,11 +452,11 @@ public final class CommonFilesPanel extends javax.swing.JPanel { org.openide.awt.Mnemonics.setLocalizedText(selectedFileCategoriesButton, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.selectedFileCategoriesButton.text")); // NOI18N selectedFileCategoriesButton.setToolTipText(org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.selectedFileCategoriesButton.toolTipText")); // NOI18N - jCheckBox1.setSelected(true); - org.openide.awt.Mnemonics.setLocalizedText(jCheckBox1, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.jCheckBox1.text")); // NOI18N + pictureVideoCheckbox.setSelected(true); + org.openide.awt.Mnemonics.setLocalizedText(pictureVideoCheckbox, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.pictureVideoCheckbox.text")); // NOI18N - jCheckBox2.setSelected(true); - org.openide.awt.Mnemonics.setLocalizedText(jCheckBox2, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.jCheckBox2.text")); // NOI18N + documentsCheckbox.setSelected(true); + org.openide.awt.Mnemonics.setLocalizedText(documentsCheckbox, org.openide.util.NbBundle.getMessage(CommonFilesPanel.class, "CommonFilesPanel.documentsCheckbox.text")); // NOI18N javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -456,50 +465,62 @@ public final class CommonFilesPanel extends javax.swing.JPanel { .addGroup(layout.createSequentialGroup() .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(selectDataSourceComboBox, javax.swing.GroupLayout.Alignment.TRAILING, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGroup(layout.createSequentialGroup() - .addGap(0, 0, Short.MAX_VALUE) - .addComponent(searchButton) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(cancelButton)) + .addGap(21, 21, 21) + .addComponent(pictureVideoCheckbox) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addGroup(layout.createSequentialGroup() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(withinDataSourceRadioButton) - .addComponent(allDataSourcesRadioButton) - .addComponent(jLabel1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addComponent(allRadioButton) - .addComponent(selectedFileCategoriesButton) .addGroup(layout.createSequentialGroup() - .addGap(21, 21, 21) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(jCheckBox2) - .addComponent(jCheckBox1)))) - .addGap(0, 54, Short.MAX_VALUE))) - .addContainerGap()) + .addComponent(withinDataSourceRadioButton) + .addComponent(allDataSourcesRadioButton) + .addComponent(commonFilesSearchLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(allRadioButton) + .addComponent(selectedFileCategoriesButton) + .addGroup(layout.createSequentialGroup() + .addGap(21, 21, 21) + .addComponent(documentsCheckbox))) + .addGap(0, 3, Short.MAX_VALUE)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addGap(0, 0, Short.MAX_VALUE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addComponent(searchButton) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(cancelButton) + .addGap(12, 12, 12)) + .addComponent(selectDataSourceComboBox, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.PREFERRED_SIZE, 261, javax.swing.GroupLayout.PREFERRED_SIZE)))) + .addContainerGap()))) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() .addContainerGap() - .addComponent(jLabel1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(commonFilesSearchLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(allDataSourcesRadioButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(withinDataSourceRadioButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(selectDataSourceComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(18, 18, 18) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(selectedFileCategoriesButton) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(jCheckBox1) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(jCheckBox2) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(allRadioButton) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(cancelButton) - .addComponent(searchButton))) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(pictureVideoCheckbox) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(documentsCheckbox) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(allRadioButton) + .addGap(0, 55, Short.MAX_VALUE)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(cancelButton) + .addComponent(searchButton)) + .addContainerGap()))) ); }// //GEN-END:initComponents @@ -542,11 +563,11 @@ public final class CommonFilesPanel extends javax.swing.JPanel { private javax.swing.JRadioButton allDataSourcesRadioButton; private javax.swing.JRadioButton allRadioButton; private javax.swing.JButton cancelButton; + private javax.swing.JLabel commonFilesSearchLabel; private javax.swing.ButtonGroup dataSourcesButtonGroup; + private javax.swing.JCheckBox documentsCheckbox; private javax.swing.ButtonGroup fileTypeFilterButtonGroup; - private javax.swing.JCheckBox jCheckBox1; - private javax.swing.JCheckBox jCheckBox2; - private javax.swing.JLabel jLabel1; + private javax.swing.JCheckBox pictureVideoCheckbox; private javax.swing.JButton searchButton; private javax.swing.JComboBox selectDataSourceComboBox; private javax.swing.JRadioButton selectedFileCategoriesButton; diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java index d2601eb025..5634a3ffb4 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleDataSource.java @@ -19,24 +19,26 @@ */ package org.sleuthkit.autopsy.commonfilesearch; +import java.util.HashSet; import java.util.Map; +import java.util.Set; /** * Provides logic for selecting common files from a single data source. */ class SingleDataSource extends CommonFilesMetaDataBuilder { - private static final String WHERE_CLAUSE = "md5 in (select md5 from tsk_files where md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL) and data_source_obj_id=%s) GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; + private static final String WHERE_CLAUSE = "md5 in (select md5 from tsk_files where md5 in (select md5 from tsk_files where (known != 1 OR known IS NULL) and data_source_obj_id=%s%s) GROUP BY md5 HAVING COUNT(*) > 1) order by md5"; private final Long selectedDataSourceId; - public SingleDataSource(Long dataSourceId, Map dataSourceIdMap) { - super(dataSourceIdMap); + public SingleDataSource(Long dataSourceId, Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType) { + super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType); this.selectedDataSourceId = dataSourceId; } @Override protected String buildSqlWhereClause() { - Object[] args = new String[]{Long.toString(this.selectedDataSourceId)}; + Object[] args = new String[]{Long.toString(this.selectedDataSourceId), determineMimeTypeFilter()}; return String.format(SingleDataSource.WHERE_CLAUSE, args); } }