Remove optional parameters from MIME types

Core/src/org/sleuthkit/autopsy/filesearch/MimeTypePanel.java

@@ -1,11 +1,23 @@
 /*
- * To change this license header, choose License Headers in Project Properties.
+ * Autopsy Forensic Browser
- * To change this template file, choose Tools | Templates
+ *
- * and open the template in the editor.
+ * Copyright 2011-2016 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package org.sleuthkit.autopsy.filesearch;
 
-import java.beans.PropertyChangeListener;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -14,18 +26,11 @@ import java.util.SortedSet;
 import java.util.logging.Level;
 import javax.swing.event.ListSelectionEvent;
 import javax.swing.event.ListSelectionListener;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.mime.MimeTypes;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
 
-/**
- *
- * @author oliver
- */
 public class MimeTypePanel extends javax.swing.JPanel {
 
-    private static final SortedSet<MediaType> mediaTypes = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes();
     private static final Logger logger = Logger.getLogger(MimeTypePanel.class.getName());
     private static final long serialVersionUID = 1L;
 
@@ -45,8 +50,8 @@ public class MimeTypePanel extends javax.swing.JPanel {
 
     private String[] getMimeTypeArray() {
         Set<String> fileTypesCollated = new HashSet<>();
-        for (MediaType mediaType : mediaTypes) {
+        for (String mediaType : FileTypeDetector.getDetectedTypes()) {
-            fileTypesCollated.add(mediaType.toString());
+            fileTypesCollated.add(mediaType);
         }
 
         FileTypeDetector fileTypeDetector;
@@ -78,7 +83,7 @@ public class MimeTypePanel extends javax.swing.JPanel {
     boolean isSelected() {
         return this.mimeTypeCheckBox.isSelected();
     }
-    
+
     void setComponentsEnabled() {
         boolean enabled = this.isSelected();
         this.mimeTypeList.setEnabled(enabled);

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeDetector.java

@@ -19,11 +19,13 @@
 package org.sleuthkit.autopsy.modules.filetypeid;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 import org.apache.tika.Tika;
-import org.apache.tika.mime.MediaType;
 import org.apache.tika.mime.MimeTypes;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
@@ -49,6 +51,7 @@ public class FileTypeDetector {
     private final byte buffer[] = new byte[BUFFER_SIZE];
     private final List<FileType> userDefinedFileTypes;
     private final List<FileType> autopsyDefinedFileTypes;
+    private static SortedSet<String> detectedTypes;    //no optional parameters
 
     /**
      * Constructs an object that detects the MIME type of a file by an
@@ -100,6 +103,21 @@ public class FileTypeDetector {
                 || isDetectableByTika(mimeType);
     }
 
+    /**
+     * Returns an unmodifiable list of MIME types that does not contain types
+     * with optional parameters. The list has no duplicate types and is in
+     * alphabetical order.
+     *
+     * @return an unmodifiable view of a set of MIME types
+     */
+    public static synchronized SortedSet<String> getDetectedTypes() {
+        if (detectedTypes == null) {
+            detectedTypes = org.apache.tika.mime.MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes()
+                    .stream().filter(t -> !t.hasParameters()).map(s -> s.toString()).collect(Collectors.toCollection(TreeSet::new));
+        }
+        return Collections.unmodifiableSortedSet(detectedTypes);
+    }
+
     /**
      * Determines whether or not a given MIME type is detectable as a
      * user-defined MIME type by this detector.
@@ -126,15 +144,7 @@ public class FileTypeDetector {
      * @return True or false.
      */
     private boolean isDetectableByTika(String mimeType) {
-        String[] split = mimeType.split("/");
+        return this.getDetectedTypes().contains(removeOptionalParameter(mimeType));
-        if (split.length == 2) {
-            String type = split[0];
-            String subtype = split[1];
-            MediaType mediaType = new MediaType(type, subtype);
-            SortedSet<MediaType> m = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes();
-            return m.contains(mediaType);
-        }
-        return false;
     }
 
     /**
@@ -196,7 +206,7 @@ public class FileTypeDetector {
          */
         String mimeType = file.getMIMEType();
         if (null != mimeType) {
-            return mimeType;
+            return removeOptionalParameter(mimeType);
         }
 
         /*
@@ -248,6 +258,7 @@ public class FileTypeDetector {
                  * Remove the Tika suffix from the MIME type name.
                  */
                 mimeType = tikaType.replace("tika-", ""); //NON-NLS
+                mimeType = removeOptionalParameter(mimeType);
 
             } catch (Exception ignored) {
                 /*
@@ -288,6 +299,20 @@ public class FileTypeDetector {
         return mimeType;
     }
 
+    /**
+     * Removes the optional parameter from a MIME type string
+     * @param  mimeType
+     * @return MIME type without the optional parameter
+     */
+    private String removeOptionalParameter(String mimeType) {
+        int indexOfSemicolon = mimeType.indexOf(";");
+        if (indexOfSemicolon != -1 ) {
+            return mimeType.substring(0, indexOfSemicolon).trim();
+        } else {
+            return mimeType;
+        }
+    }
+
     /**
      * Determines whether or not the a file matches a user-defined custom file
      * type.

Core/src/org/sleuthkit/autopsy/modules/filetypeid/TikaFileTypeDetector.java

@@ -24,9 +24,6 @@ import org.apache.tika.mime.MediaType;
 import org.apache.tika.mime.MimeTypes;
 
 import org.sleuthkit.datamodel.AbstractFile;
-import org.sleuthkit.datamodel.BlackboardArtifact;
-import org.sleuthkit.datamodel.BlackboardAttribute;
-import org.sleuthkit.datamodel.TskCoreException;
 
 /**
  * @deprecated Use org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector

Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java

@@ -31,8 +31,6 @@ import java.util.regex.PatternSyntaxException;
 import javax.swing.JButton;
 import javax.swing.JComponent;
 import javax.swing.JOptionPane;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.mime.MimeTypes;
 import org.openide.DialogDisplayer;
 import org.openide.NotifyDescriptor;
 import org.openide.util.NbBundle;
@@ -58,7 +56,6 @@ final class FilesSetRulePanel extends javax.swing.JPanel {
         "FilesSetRulePanel.ZeroFileSizeError=File size condition value must not be 0 (Unless = is selected)."
     })
 
-    private static final SortedSet<MediaType> mediaTypes = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes();
     private static final Logger logger = Logger.getLogger(FilesSetRulePanel.class.getName());
     private static final String SLEUTHKIT_PATH_SEPARATOR = "/"; // NON-NLS
     private static final List<String> ILLEGAL_FILE_NAME_CHARS = InterestingItemDefsManager.getIllegalFileNameChars();
@@ -106,8 +103,8 @@ final class FilesSetRulePanel extends javax.swing.JPanel {
 
     private void populateMimeTypesComboBox() {
         Set<String> fileTypesCollated = new HashSet<>();
-        for (MediaType mediaType : mediaTypes) {
+        for (String mediaType : FileTypeDetector.getDetectedTypes()) {
-            fileTypesCollated.add(mediaType.toString());
+            fileTypesCollated.add(mediaType);
         }
 
         FileTypeDetector fileTypeDetector;
@@ -321,7 +318,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel {
             }
         }
 
-        // The path condition, if specified, must either be a regular expression 
+        // The path condition, if specified, must either be a regular expression
         // that compiles or a string without illegal file path chars.
         if (this.pathCheck.isSelected()) {
             if (this.pathTextField.getText().isEmpty()) {
@@ -412,7 +409,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel {
                 }
             } else {
                 logger.log(Level.SEVERE, "Attempt to get name condition with illegal chars"); // NON-NLS
-                throw new IllegalStateException("The files set rule panel name condition is not in a valid state"); // NON-NLS                    
+                throw new IllegalStateException("The files set rule panel name condition is not in a valid state"); // NON-NLS
             }
         }
         return condition;
@@ -498,7 +495,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel {
                     condition = new FilesSet.Rule.ParentPathCondition(path);
                 } else {
                     logger.log(Level.SEVERE, "Attempt to get path condition with illegal chars"); // NON-NLS
-                    throw new IllegalStateException("The files set rule panel path condition is not in a valid state"); // NON-NLS                    
+                    throw new IllegalStateException("The files set rule panel path condition is not in a valid state"); // NON-NLS
                 }
             }
         }

Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java

@@ -34,7 +34,6 @@ import javax.swing.JOptionPane;
 import javax.swing.event.ListSelectionEvent;
 import javax.swing.event.ListSelectionListener;
 import org.apache.tika.mime.MediaType;
-import org.apache.tika.mime.MimeTypes;
 import org.netbeans.spi.options.OptionsPanelController;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.corecomponents.OptionsPanel;
@@ -57,16 +56,15 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
         "InterestingItemsDefsPanel.saveError=Error saving interesting files sets to file."
     })
 
-    private static final SortedSet<MediaType> mediaTypes = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes();
     private final DefaultListModel<FilesSet> setsListModel = new DefaultListModel<>();
     private final DefaultListModel<FilesSet.Rule> rulesListModel = new DefaultListModel<>();
     private final Logger logger = Logger.getLogger(InterestingItemDefsPanel.class.getName());
-    private JButton okButton = new JButton("OK");
+    private final JButton okButton = new JButton("OK");
-    private JButton cancelButton = new JButton("Cancel");
+    private final JButton cancelButton = new JButton("Cancel");
 
-    // The following is a map of interesting files set names to interesting 
+    // The following is a map of interesting files set names to interesting
-    // files set definitions. It is a snapshot of the files set definitions 
+    // files set definitions. It is a snapshot of the files set definitions
-    // obtained from the interesting item definitions manager at the time the 
+    // obtained from the interesting item definitions manager at the time the
     // the panel is loaded. When the panel saves or stores its settings, these
     // definitions, possibly changed, are submitted back to the interesting item
     // definitions manager. Note that it is a tree map to aid in displaying
@@ -88,10 +86,10 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
     @NbBundle.Messages({"InterestingItemDefsPanel.Title=Global Interesting Items Settings"})
     private void customInit() {
         setName(Bundle.InterestingItemDefsPanel_Title());
-        
+
         Set<String> fileTypesCollated = new HashSet<>();
-        for (MediaType mediaType : mediaTypes) {
+        for (String mediaType : FileTypeDetector.getDetectedTypes()) {
-            fileTypesCollated.add(mediaType.toString());
+            fileTypesCollated.add(mediaType);
         }
 
         FileTypeDetector fileTypeDetector;
@@ -119,7 +117,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
         this.fileSizeUnitComboBox.setSelectedIndex(1);
         this.equalitySignComboBox.setSelectedIndex(2);
     }
-    
+
     /**
      * @inheritDoc
      */
@@ -156,14 +154,14 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             this.filesSets = new TreeMap<>();
         }
 
-        // Populate the list model for the interesting files sets list 
+        // Populate the list model for the interesting files sets list
         // component.
         for (FilesSet set : this.filesSets.values()) {
             this.setsListModel.addElement(set);
         }
 
         if (!this.filesSets.isEmpty()) {
-            // Select the first files set by default. The list selections 
+            // Select the first files set by default. The list selections
             // listeners will then populate the other components.
             EventQueue.invokeLater(() -> {
                 InterestingItemDefsPanel.this.setsList.setSelectedIndex(0);
@@ -222,7 +220,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             // components.
             FilesSet selectedSet = InterestingItemDefsPanel.this.setsList.getSelectedValue();
             if (selectedSet != null) {
-                // Populate the components that display the properties of the 
+                // Populate the components that display the properties of the
                 // selected files set.
                 InterestingItemDefsPanel.this.setDescriptionTextArea.setText(selectedSet.getDescription());
                 InterestingItemDefsPanel.this.ignoreKnownFilesCheckbox.setSelected(selectedSet.ignoresKnownFiles());
@@ -269,7 +267,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
                 FilesSet.Rule.MimeTypeCondition mimeTypeCondition = rule.getMimeTypeCondition();
                 FilesSet.Rule.FileSizeCondition fileSizeCondition = rule.getFileSizeCondition();
 
-                // Populate the components that display the properties of the 
+                // Populate the components that display the properties of the
                 // selected rule.
                 if (nameCondition != null) {
                     InterestingItemDefsPanel.this.fileNameTextField.setText(nameCondition.getTextToMatch());
@@ -345,7 +343,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             panel = new FilesSetPanel();
         }
 
-        // Do a dialog box with the files set panel until the user either enters 
+        // Do a dialog box with the files set panel until the user either enters
         // a valid definition or cancels. Note that the panel gives the user
         // feedback when isValidDefinition() is called.
         int option = JOptionPane.OK_OPTION;
@@ -366,7 +364,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             Map<String, FilesSet.Rule> rules = new HashMap<>();
             if (selectedSet != null) {
                 // Interesting file sets are immutable for thread safety,
-                // so editing a files set definition is a replacement operation. 
+                // so editing a files set definition is a replacement operation.
                 // Preserve the existing rules from the set being edited.
                 rules.putAll(selectedSet.getRules());
             }
@@ -391,7 +389,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             // Creating a new rule definition.
             panel = new FilesSetRulePanel(okButton, cancelButton);
         }
-        // Do a dialog box with the files set panel until the user either enters 
+        // Do a dialog box with the files set panel until the user either enters
         // a valid definition or cancels. Note that the panel gives the user
         // feedback when isValidDefinition() is called.
         int option = JOptionPane.OK_OPTION;
@@ -401,12 +399,12 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
 
         if (option == JOptionPane.OK_OPTION) {
             // Interesting file sets are immutable for thread safety,
-            // so editing a files set rule definition is a replacement 
+            // so editing a files set rule definition is a replacement
             // operation. Preserve the existing rules from the set being edited.
             FilesSet selectedSet = this.setsList.getSelectedValue();
             Map<String, FilesSet.Rule> rules = new HashMap<>(selectedSet.getRules());
 
-            // Remove the "old" rule definition and add the new/edited 
+            // Remove the "old" rule definition and add the new/edited
             // definition.
             if (selectedRule != null) {
                 rules.remove(selectedRule.getUuid());
@@ -414,12 +412,12 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             FilesSet.Rule newRule = new FilesSet.Rule(panel.getRuleName(), panel.getFileNameCondition(), panel.getMetaTypeCondition(), panel.getPathCondition(), panel.getMimeTypeCondition(), panel.getFileSizeCondition());
             rules.put(newRule.getUuid(), newRule);
 
-            // Add the new/edited files set definition, replacing any previous 
+            // Add the new/edited files set definition, replacing any previous
             // definition with the same name and refreshing the display.
             this.replaceFilesSet(selectedSet, selectedSet.getName(), selectedSet.getDescription(), selectedSet.ignoresKnownFiles(), rules);
 
-            // Select the new/edited rule. Queue it up so it happens after the 
+            // Select the new/edited rule. Queue it up so it happens after the
-            // selection listeners react to the selection of the "new" files 
+            // selection listeners react to the selection of the "new" files
             // set.
             EventQueue.invokeLater(() -> {
                 this.rulesList.setSelectedValue(newRule, true);
@@ -459,8 +457,8 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             this.setsListModel.addElement(set);
         }
 
-        // Select the new/edited files set definition in the set definitions 
+        // Select the new/edited files set definition in the set definitions
-        // list. This will cause the selection listeners to repopulate the 
+        // list. This will cause the selection listeners to repopulate the
         // subordinate components.
         this.setsList.setSelectedValue(newSet, true);
     }