From e4f351bf8cc42d5631ed8a88a48b724f924d39d7 Mon Sep 17 00:00:00 2001 From: Sophie Mori Date: Mon, 21 Nov 2016 17:06:31 -0500 Subject: [PATCH] Remove optional parameters from MIME types --- .../autopsy/filesearch/MimeTypePanel.java | 33 +++++++------ .../modules/filetypeid/FileTypeDetector.java | 47 +++++++++++++----- .../filetypeid/TikaFileTypeDetector.java | 3 -- .../interestingitems/FilesSetRulePanel.java | 13 ++--- .../InterestingItemDefsPanel.java | 48 +++++++++---------- 5 files changed, 83 insertions(+), 61 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/MimeTypePanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/MimeTypePanel.java index dfebc0c718..0949e5f8fe 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/MimeTypePanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/MimeTypePanel.java @@ -1,11 +1,23 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2011-2016 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.filesearch; -import java.beans.PropertyChangeListener; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -14,18 +26,11 @@ import java.util.SortedSet; import java.util.logging.Level; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; -import org.apache.tika.mime.MediaType; -import org.apache.tika.mime.MimeTypes; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; -/** - * - * @author oliver - */ public class MimeTypePanel extends javax.swing.JPanel { - private static final SortedSet mediaTypes = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); private static final Logger logger = Logger.getLogger(MimeTypePanel.class.getName()); private static final long serialVersionUID = 1L; @@ -45,8 +50,8 @@ public class MimeTypePanel extends javax.swing.JPanel { private String[] getMimeTypeArray() { Set fileTypesCollated = new HashSet<>(); - for (MediaType mediaType : mediaTypes) { - fileTypesCollated.add(mediaType.toString()); + for (String mediaType : FileTypeDetector.getDetectedTypes()) { + fileTypesCollated.add(mediaType); } FileTypeDetector fileTypeDetector; @@ -78,7 +83,7 @@ public class MimeTypePanel extends javax.swing.JPanel { boolean isSelected() { return this.mimeTypeCheckBox.isSelected(); } - + void setComponentsEnabled() { boolean enabled = this.isSelected(); this.mimeTypeList.setEnabled(enabled); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java index b0f95c93fa..42bdfbe619 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java @@ -19,11 +19,13 @@ package org.sleuthkit.autopsy.modules.filetypeid; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.SortedSet; +import java.util.TreeSet; import java.util.logging.Level; +import java.util.stream.Collectors; import org.apache.tika.Tika; -import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MimeTypes; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; @@ -49,6 +51,7 @@ public class FileTypeDetector { private final byte buffer[] = new byte[BUFFER_SIZE]; private final List userDefinedFileTypes; private final List autopsyDefinedFileTypes; + private static SortedSet detectedTypes; //no optional parameters /** * Constructs an object that detects the MIME type of a file by an @@ -100,6 +103,21 @@ public class FileTypeDetector { || isDetectableByTika(mimeType); } + /** + * Returns an unmodifiable list of MIME types that does not contain types + * with optional parameters. The list has no duplicate types and is in + * alphabetical order. + * + * @return an unmodifiable view of a set of MIME types + */ + public static synchronized SortedSet getDetectedTypes() { + if (detectedTypes == null) { + detectedTypes = org.apache.tika.mime.MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes() + .stream().filter(t -> !t.hasParameters()).map(s -> s.toString()).collect(Collectors.toCollection(TreeSet::new)); + } + return Collections.unmodifiableSortedSet(detectedTypes); + } + /** * Determines whether or not a given MIME type is detectable as a * user-defined MIME type by this detector. @@ -126,15 +144,7 @@ public class FileTypeDetector { * @return True or false. */ private boolean isDetectableByTika(String mimeType) { - String[] split = mimeType.split("/"); - if (split.length == 2) { - String type = split[0]; - String subtype = split[1]; - MediaType mediaType = new MediaType(type, subtype); - SortedSet m = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); - return m.contains(mediaType); - } - return false; + return this.getDetectedTypes().contains(removeOptionalParameter(mimeType)); } /** @@ -196,7 +206,7 @@ public class FileTypeDetector { */ String mimeType = file.getMIMEType(); if (null != mimeType) { - return mimeType; + return removeOptionalParameter(mimeType); } /* @@ -248,6 +258,7 @@ public class FileTypeDetector { * Remove the Tika suffix from the MIME type name. */ mimeType = tikaType.replace("tika-", ""); //NON-NLS + mimeType = removeOptionalParameter(mimeType); } catch (Exception ignored) { /* @@ -288,6 +299,20 @@ public class FileTypeDetector { return mimeType; } + /** + * Removes the optional parameter from a MIME type string + * @param mimeType + * @return MIME type without the optional parameter + */ + private String removeOptionalParameter(String mimeType) { + int indexOfSemicolon = mimeType.indexOf(";"); + if (indexOfSemicolon != -1 ) { + return mimeType.substring(0, indexOfSemicolon).trim(); + } else { + return mimeType; + } + } + /** * Determines whether or not the a file matches a user-defined custom file * type. diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java index e8bd4ab29b..c237fe6222 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java @@ -24,9 +24,6 @@ import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MimeTypes; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.TskCoreException; /** * @deprecated Use org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java index 9ee7f1a172..4d249f35ce 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java @@ -31,8 +31,6 @@ import java.util.regex.PatternSyntaxException; import javax.swing.JButton; import javax.swing.JComponent; import javax.swing.JOptionPane; -import org.apache.tika.mime.MediaType; -import org.apache.tika.mime.MimeTypes; import org.openide.DialogDisplayer; import org.openide.NotifyDescriptor; import org.openide.util.NbBundle; @@ -58,7 +56,6 @@ final class FilesSetRulePanel extends javax.swing.JPanel { "FilesSetRulePanel.ZeroFileSizeError=File size condition value must not be 0 (Unless = is selected)." }) - private static final SortedSet mediaTypes = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); private static final Logger logger = Logger.getLogger(FilesSetRulePanel.class.getName()); private static final String SLEUTHKIT_PATH_SEPARATOR = "/"; // NON-NLS private static final List ILLEGAL_FILE_NAME_CHARS = InterestingItemDefsManager.getIllegalFileNameChars(); @@ -106,8 +103,8 @@ final class FilesSetRulePanel extends javax.swing.JPanel { private void populateMimeTypesComboBox() { Set fileTypesCollated = new HashSet<>(); - for (MediaType mediaType : mediaTypes) { - fileTypesCollated.add(mediaType.toString()); + for (String mediaType : FileTypeDetector.getDetectedTypes()) { + fileTypesCollated.add(mediaType); } FileTypeDetector fileTypeDetector; @@ -321,7 +318,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel { } } - // The path condition, if specified, must either be a regular expression + // The path condition, if specified, must either be a regular expression // that compiles or a string without illegal file path chars. if (this.pathCheck.isSelected()) { if (this.pathTextField.getText().isEmpty()) { @@ -412,7 +409,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel { } } else { logger.log(Level.SEVERE, "Attempt to get name condition with illegal chars"); // NON-NLS - throw new IllegalStateException("The files set rule panel name condition is not in a valid state"); // NON-NLS + throw new IllegalStateException("The files set rule panel name condition is not in a valid state"); // NON-NLS } } return condition; @@ -498,7 +495,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel { condition = new FilesSet.Rule.ParentPathCondition(path); } else { logger.log(Level.SEVERE, "Attempt to get path condition with illegal chars"); // NON-NLS - throw new IllegalStateException("The files set rule panel path condition is not in a valid state"); // NON-NLS + throw new IllegalStateException("The files set rule panel path condition is not in a valid state"); // NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java index ea1b2f2fdb..c3f4b949d1 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java @@ -34,7 +34,6 @@ import javax.swing.JOptionPane; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; import org.apache.tika.mime.MediaType; -import org.apache.tika.mime.MimeTypes; import org.netbeans.spi.options.OptionsPanelController; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; @@ -57,16 +56,15 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp "InterestingItemsDefsPanel.saveError=Error saving interesting files sets to file." }) - private static final SortedSet mediaTypes = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); private final DefaultListModel setsListModel = new DefaultListModel<>(); private final DefaultListModel rulesListModel = new DefaultListModel<>(); private final Logger logger = Logger.getLogger(InterestingItemDefsPanel.class.getName()); - private JButton okButton = new JButton("OK"); - private JButton cancelButton = new JButton("Cancel"); + private final JButton okButton = new JButton("OK"); + private final JButton cancelButton = new JButton("Cancel"); - // The following is a map of interesting files set names to interesting - // files set definitions. It is a snapshot of the files set definitions - // obtained from the interesting item definitions manager at the time the + // The following is a map of interesting files set names to interesting + // files set definitions. It is a snapshot of the files set definitions + // obtained from the interesting item definitions manager at the time the // the panel is loaded. When the panel saves or stores its settings, these // definitions, possibly changed, are submitted back to the interesting item // definitions manager. Note that it is a tree map to aid in displaying @@ -88,10 +86,10 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp @NbBundle.Messages({"InterestingItemDefsPanel.Title=Global Interesting Items Settings"}) private void customInit() { setName(Bundle.InterestingItemDefsPanel_Title()); - + Set fileTypesCollated = new HashSet<>(); - for (MediaType mediaType : mediaTypes) { - fileTypesCollated.add(mediaType.toString()); + for (String mediaType : FileTypeDetector.getDetectedTypes()) { + fileTypesCollated.add(mediaType); } FileTypeDetector fileTypeDetector; @@ -119,7 +117,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp this.fileSizeUnitComboBox.setSelectedIndex(1); this.equalitySignComboBox.setSelectedIndex(2); } - + /** * @inheritDoc */ @@ -156,14 +154,14 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp this.filesSets = new TreeMap<>(); } - // Populate the list model for the interesting files sets list + // Populate the list model for the interesting files sets list // component. for (FilesSet set : this.filesSets.values()) { this.setsListModel.addElement(set); } if (!this.filesSets.isEmpty()) { - // Select the first files set by default. The list selections + // Select the first files set by default. The list selections // listeners will then populate the other components. EventQueue.invokeLater(() -> { InterestingItemDefsPanel.this.setsList.setSelectedIndex(0); @@ -222,7 +220,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp // components. FilesSet selectedSet = InterestingItemDefsPanel.this.setsList.getSelectedValue(); if (selectedSet != null) { - // Populate the components that display the properties of the + // Populate the components that display the properties of the // selected files set. InterestingItemDefsPanel.this.setDescriptionTextArea.setText(selectedSet.getDescription()); InterestingItemDefsPanel.this.ignoreKnownFilesCheckbox.setSelected(selectedSet.ignoresKnownFiles()); @@ -269,7 +267,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp FilesSet.Rule.MimeTypeCondition mimeTypeCondition = rule.getMimeTypeCondition(); FilesSet.Rule.FileSizeCondition fileSizeCondition = rule.getFileSizeCondition(); - // Populate the components that display the properties of the + // Populate the components that display the properties of the // selected rule. if (nameCondition != null) { InterestingItemDefsPanel.this.fileNameTextField.setText(nameCondition.getTextToMatch()); @@ -345,7 +343,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp panel = new FilesSetPanel(); } - // Do a dialog box with the files set panel until the user either enters + // Do a dialog box with the files set panel until the user either enters // a valid definition or cancels. Note that the panel gives the user // feedback when isValidDefinition() is called. int option = JOptionPane.OK_OPTION; @@ -366,7 +364,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp Map rules = new HashMap<>(); if (selectedSet != null) { // Interesting file sets are immutable for thread safety, - // so editing a files set definition is a replacement operation. + // so editing a files set definition is a replacement operation. // Preserve the existing rules from the set being edited. rules.putAll(selectedSet.getRules()); } @@ -391,7 +389,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp // Creating a new rule definition. panel = new FilesSetRulePanel(okButton, cancelButton); } - // Do a dialog box with the files set panel until the user either enters + // Do a dialog box with the files set panel until the user either enters // a valid definition or cancels. Note that the panel gives the user // feedback when isValidDefinition() is called. int option = JOptionPane.OK_OPTION; @@ -401,12 +399,12 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp if (option == JOptionPane.OK_OPTION) { // Interesting file sets are immutable for thread safety, - // so editing a files set rule definition is a replacement + // so editing a files set rule definition is a replacement // operation. Preserve the existing rules from the set being edited. FilesSet selectedSet = this.setsList.getSelectedValue(); Map rules = new HashMap<>(selectedSet.getRules()); - // Remove the "old" rule definition and add the new/edited + // Remove the "old" rule definition and add the new/edited // definition. if (selectedRule != null) { rules.remove(selectedRule.getUuid()); @@ -414,12 +412,12 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp FilesSet.Rule newRule = new FilesSet.Rule(panel.getRuleName(), panel.getFileNameCondition(), panel.getMetaTypeCondition(), panel.getPathCondition(), panel.getMimeTypeCondition(), panel.getFileSizeCondition()); rules.put(newRule.getUuid(), newRule); - // Add the new/edited files set definition, replacing any previous + // Add the new/edited files set definition, replacing any previous // definition with the same name and refreshing the display. this.replaceFilesSet(selectedSet, selectedSet.getName(), selectedSet.getDescription(), selectedSet.ignoresKnownFiles(), rules); - // Select the new/edited rule. Queue it up so it happens after the - // selection listeners react to the selection of the "new" files + // Select the new/edited rule. Queue it up so it happens after the + // selection listeners react to the selection of the "new" files // set. EventQueue.invokeLater(() -> { this.rulesList.setSelectedValue(newRule, true); @@ -459,8 +457,8 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp this.setsListModel.addElement(set); } - // Select the new/edited files set definition in the set definitions - // list. This will cause the selection listeners to repopulate the + // Select the new/edited files set definition in the set definitions + // list. This will cause the selection listeners to repopulate the // subordinate components. this.setsList.setSelectedValue(newSet, true); }