Merge pull request #420 from rcordovano/merge_rc_branch

Merge rc branch

Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java

@@ -126,6 +126,36 @@ public class BlackboardArtifactNode extends DisplayableItemNode {
         }
         final int artifactTypeId = artifact.getArtifactTypeID();
         
+        // If mismatch, add props for extension and file type
+        if (artifactTypeId == BlackboardArtifact.ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED.getTypeID()) {
+            String actualExt = "";
+            int i = associated.getName().lastIndexOf(".");
+            if ((i > -1) && ((i + 1) < associated.getName().length())) {
+                actualExt = associated.getName().substring(i + 1).toLowerCase();
+            }                    
+            ss.put(new NodeProperty("Extension", "Extension", NO_DESCR, actualExt));        
+        
+            try {
+                String actualMimeType = "";
+                ArrayList<BlackboardArtifact> artList = associated.getAllArtifacts();
+                for (BlackboardArtifact art : artList) {
+                    List<BlackboardAttribute> atrList = art.getAttributes();
+                    for (BlackboardAttribute att : atrList) {
+                        if (att.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID()) {                        
+                            actualMimeType = att.getValueString();
+                        }
+                    }                
+                }
+                if (actualMimeType.isEmpty()) {
+                    logger.log(Level.WARNING, "Could not find expected TSK_FILE_TYPE_SIG attribute.");
+                } else {
+                    ss.put(new NodeProperty("MIME Type", "MIME Type", NO_DESCR, actualMimeType));
+                }
+            } catch (TskCoreException ex) {
+                logger.log(Level.WARNING, "Error while searching for TSK_FILE_TYPE_SIG attribute: ", ex);
+            }            
+        }        
+        
         if (Arrays.asList(SHOW_UNIQUE_PATH).contains(artifactTypeId)) {
             String sourcePath = "";
             try {

Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java

@@ -19,6 +19,7 @@
 package org.sleuthkit.autopsy.ingest;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import javax.swing.JPanel;
 import org.openide.util.lookup.ServiceProvider;
@@ -29,6 +30,7 @@ import org.sleuthkit.datamodel.Content;
 public class GeneralIngestConfigurator implements IngestConfigurator { 
    
     public static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules";
+    public static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules";
     public static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space";
     private List<Content> contentToIngest;
     private IngestManager manager;
@@ -51,20 +53,57 @@ public class GeneralIngestConfigurator implements IngestConfigurator {
 
     private List<String> loadSettingsForContext() {        
         List<String> messages = new ArrayList<>();  
+        List<IngestModuleAbstract> allModules = IngestManager.getDefault().enumerateAllModules();
         
         // If there is no enabled ingest modules setting for this user, default to enabling all
         // of the ingest modules the IngestManager has loaded.
         if (ModuleSettings.settingExists(moduleContext, ENABLED_INGEST_MODULES_KEY) == false) {
-            String defaultSetting = moduleListToCsv(IngestManager.getDefault().enumerateAllModules());
+            String defaultSetting = moduleListToCsv(allModules);
             ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, defaultSetting);
         }        
         
+        String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", ");
+        ArrayList<String> enabledList = new ArrayList<>(Arrays.asList(enabledModuleNames));
+        
+        // Check for modules that are missing from the config file
+        
+        String[] disabledModuleNames = null;
+        // Older config files won't have the disabled list, so don't assume it exists
+        if (ModuleSettings.settingExists(moduleContext, DISABLED_INGEST_MODULES_KEY)) {
+            disabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY).split(", ");
+        }
+        
+        for (IngestModuleAbstract module : allModules) {
+            boolean found = false;
+
+            // Check enabled first
+            for (String moduleName : enabledModuleNames) {
+                if (module.getName().equals(moduleName)) {
+                    found = true;
+                    break;
+                }
+            }                
+
+            // Then check disabled
+            if (!found && (disabledModuleNames != null)) {
+                for (String moduleName : disabledModuleNames) {
+                    if (module.getName().equals(moduleName)) {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+
+            if (!found) {
+                enabledList.add(module.getName());
+                // It will get saved to file later
+            }
+        }            
+        
         // Get the enabled ingest modules setting, check for missing modules, and pass the setting to
         // the UI component.
-        List<IngestModuleAbstract> allModules = IngestManager.getDefault().enumerateAllModules();
-        String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", ");
         List<IngestModuleAbstract> enabledModules = new ArrayList<>();
-        for (String moduleName : enabledModuleNames) {
+        for (String moduleName : enabledList) {
             if (moduleName.equals("Thunderbird Parser") 
                     || moduleName.equals("MBox Parser")) {
                 moduleName = "Email Parser";
@@ -112,6 +151,10 @@ public class GeneralIngestConfigurator implements IngestConfigurator {
         String enabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getModulesToStart());
         ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, enabledModulesCsvList);
         
+        // Save the user's configuration of the set of disabled ingest modules.
+        String disabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getDisabledModules());
+        ModuleSettings.setConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY, disabledModulesCsvList);        
+        
         // Save the user's setting for the process unallocated space flag.
         String processUnalloc = Boolean.toString(ingestDialogPanel.processUnallocSpaceEnabled());
         ModuleSettings.setConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY, processUnalloc);

Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java

@@ -60,7 +60,7 @@ public class IngestDialogPanel extends javax.swing.JPanel {
         this.context = context;
     }
     
-
+    
     public IngestModuleAbstract getCurrentIngestModule() {
         return currentModule;
     }
@@ -69,6 +69,10 @@ public class IngestDialogPanel extends javax.swing.JPanel {
         return tableModel.getSelectedModules();
     }
     
+    public List<IngestModuleAbstract> getDisabledModules() {
+        return tableModel.getUnSelectedModules();
+    }
+    
     public boolean processUnallocSpaceEnabled() {
         return processUnallocCheckbox.isSelected();
     }
@@ -350,6 +354,16 @@ public class IngestDialogPanel extends javax.swing.JPanel {
             return selectedModules;
         }
         
+        public List<IngestModuleAbstract> getUnSelectedModules() {
+            List<IngestModuleAbstract> unselectedModules = new ArrayList<>();
+            for (Map.Entry<IngestModuleAbstract, Boolean> entry : moduleData) {
+                if (!entry.getValue().booleanValue()) {
+                    unselectedModules.add(entry.getKey());
+                }
+            }
+            return unselectedModules;
+        }        
+        
         /**
          * Sets the given modules as selected in the modules table
          * @param selectedModules 

FileTypeId/nbproject/project.xml

@@ -91,7 +91,9 @@
                     </run-dependency>
                 </dependency>
             </module-dependencies>
-            <public-packages/>
+            <public-packages>
+                <package>org.sleuthkit.autopsy.filetypeid</package>
+            </public-packages>
             <class-path-extension>
                 <runtime-relative-path>ext/tika-core-1.2.jar</runtime-relative-path>
                 <binary-origin>release/modules/ext/tika-core-1.2.jar</binary-origin>

FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java

@@ -36,5 +36,7 @@ public interface FileTypeDetectionInterface {
     }
     
     // You only have one job
-    FileIdInfo attemptMatch(AbstractFile abstractFile);    
+    FileIdInfo attemptMatch(AbstractFile abstractFile);
+    
+    boolean isMimeTypeDetectable(String mimeType);
 }

FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java

@@ -48,7 +48,7 @@ public class FileTypeIdIngestModule extends org.sleuthkit.autopsy.ingest.IngestM
     private static long matchTime = 0;
     private static int messageId = 0;
     private static long numFiles = 0;
-    private static boolean skipKnown = false;
+    private static boolean skipKnown = true;
     
     private FileTypeIdSimpleConfigPanel simpleConfigPanel;
     private IngestServices services;
@@ -173,4 +173,15 @@ public class FileTypeIdIngestModule extends org.sleuthkit.autopsy.ingest.IngestM
     public static void setSkipKnown(boolean flag) {
         skipKnown = flag;
     }
+    
+    /**
+     * Validate if a given mime type is in the detector's registry.
+     * @param mimeType Full string of mime type, e.g. "text/html"
+     * @return true if detectable
+     */
+    public static boolean isMimeTypeDetectable(String mimeType) {
+        FileTypeDetectionInterface detector = new TikaFileTypeDetector();         
+        return detector.isMimeTypeDetectable(mimeType);
+    }    
+    
 }

FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form

@@ -36,6 +36,7 @@
   <SubComponents>
     <Component class="javax.swing.JCheckBox" name="skipKnownCheckBox">
       <Properties>
+        <Property name="selected" type="boolean" value="true"/>
         <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
           <ResourceString bundle="org/sleuthkit/autopsy/filetypeid/Bundle.properties" key="FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
         </Property>

FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java

@@ -44,6 +44,7 @@ public class FileTypeIdSimpleConfigPanel extends javax.swing.JPanel {
 
         skipKnownCheckBox = new javax.swing.JCheckBox();
 
+        skipKnownCheckBox.setSelected(true);
         skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text")); // NOI18N
         skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.toolTipText")); // NOI18N
         skipKnownCheckBox.addActionListener(new java.awt.event.ActionListener() {

FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java

@@ -18,8 +18,11 @@
  */
 
 package org.sleuthkit.autopsy.filetypeid;
+import java.util.SortedSet;
 import org.openide.util.Exceptions;
 import org.apache.tika.Tika;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeTypes;
 
 import org.sleuthkit.datamodel.AbstractFile;
 
@@ -52,4 +55,27 @@ public class TikaFileTypeDetector implements FileTypeDetectionInterface {
         }        
     }
 
+    /**
+     * Validate if a given mime type is in the registry.
+     * For Tika, we remove the string "tika" from all MIME names, 
+     * e.g. use "application/x-msoffice" NOT "application/x-tika-msoffice"
+     * @param mimeType Full string of mime type, e.g. "text/html"
+     * @return true if detectable
+     */
+    @Override
+    public boolean isMimeTypeDetectable(String mimeType) {
+        boolean ret = false;
+        
+        SortedSet<MediaType> m = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes();        
+        String[] split = mimeType.split("/");
+        
+        if (split.length == 2) {
+            String type = split[0];
+            String subtype = split[1];
+            MediaType mediaType = new MediaType(type, subtype);
+            ret = m.contains(mediaType);
+        }
+
+        return ret;        
+    }
 }

HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java

@@ -50,6 +50,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile {
     public final static String MODULE_DESCRIPTION = "Identifies known and notables files using supplied hash databases, such as a standard NSRL database.";
     final public static String MODULE_VERSION = Version.getVersion();
     private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
+    private static final int MAX_COMMENT_SIZE = 500;
     private HashDbSimpleConfigPanel simpleConfigPanel;
     private HashDbConfigPanel advancedConfigPanel;
     private IngestServices services;
@@ -227,9 +228,24 @@ public class HashDbIngestModule extends IngestModuleAbstractFile {
                         services.postMessage(IngestMessage.createErrorMessage(++messageId, HashDbIngestModule.this, "Hash Lookup Error: " + name,
                                 "Error encountered while setting known bad state for " + name + "."));
                         ret = ProcessResult.ERROR;
-                    }
+                    }                    
                     String hashSetName = db.getHashSetName();
-                    postHashSetHitToBlackboard(file, md5Hash, hashSetName, db.getSendIngestMessages());
+                    
+                    String comment = "";                    
+                    ArrayList<String> comments = db.lookUp(file).getComments();
+                    int i = 0;
+                    for (String c : comments) {
+                        comment += c;
+                        if (++i > 1) {
+                            c += ". ";
+                        }
+                        if (comment.length() > MAX_COMMENT_SIZE) {
+                            comment = comment.substring(0, MAX_COMMENT_SIZE) + "...";
+                            break;
+                        }                        
+                    }
+
+                    postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
                 }
                 lookuptime += (System.currentTimeMillis() - lookupstart);
             } catch (TskException ex) {
@@ -271,7 +287,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile {
         return ret;
     }
 
-    private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, boolean showInboxMessage) {
+    private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) {
         try {
             BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT);
             //TODO Revisit usage of deprecated constructor as per TSK-583
@@ -280,6 +296,9 @@ public class HashDbIngestModule extends IngestModuleAbstractFile {
             badFile.addAttribute(att2);
             BlackboardAttribute att3 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5.getTypeID(), MODULE_NAME, md5Hash);
             badFile.addAttribute(att3);
+            BlackboardAttribute att4 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT.getTypeID(), MODULE_NAME, comment);
+            badFile.addAttribute(att4);
+            
             if (showInboxMessage) {
                 StringBuilder detailsSb = new StringBuilder();
                 //details

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2013 Basis Technology Corp.
+ * Copyright 2011-2014 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +22,6 @@ import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
 import java.io.IOException;
 import java.io.InputStream;
-import java.lang.Long;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -128,7 +127,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile {
     private static List<AbstractFileExtract> textExtractors;
     private static AbstractFileStringExtract stringExtractor;
     private boolean initialized = false;
-    private KeywordSearchConfigurationPanel panel;
+    private KeywordSearchIngestSimplePanel simpleConfigPanel;
+    private KeywordSearchConfigurationPanel advancedConfigPanel;
     private Tika tikaFormatDetector;
     
 
@@ -436,26 +436,36 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile {
     @Override
     public javax.swing.JPanel getSimpleConfiguration(String context) {
         KeywordSearchListsXML.getCurrent().reload();
-        return new KeywordSearchIngestSimplePanel();
+        
+        if (null == simpleConfigPanel) {
+           simpleConfigPanel = new KeywordSearchIngestSimplePanel();  
+        }
+        else {
+            simpleConfigPanel.load();
+        }
+        
+        return simpleConfigPanel;
     }
 
     @Override
     public javax.swing.JPanel getAdvancedConfiguration(String context) {
-        //return KeywordSearchConfigurationPanel.getDefault();
+        if (advancedConfigPanel == null) {
-        getPanel().load();
+            advancedConfigPanel = new KeywordSearchConfigurationPanel();
-        return getPanel();
-    }
-
-    private KeywordSearchConfigurationPanel getPanel() {
-        if (panel == null) {
-            panel = new KeywordSearchConfigurationPanel();
         }
-        return panel;
+        
+        advancedConfigPanel.load();
+        return advancedConfigPanel;
     }
 
     @Override
     public void saveAdvancedConfiguration() {
-        getPanel().store();
+        if (advancedConfigPanel != null) {
+            advancedConfigPanel.store();
+        }
+        
+        if (simpleConfigPanel != null) {
+            simpleConfigPanel.load();
+        }
     }
 
     @Override

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java

@@ -70,10 +70,12 @@ public class KeywordSearchIngestSimplePanel extends javax.swing.JPanel {
         reloadEncodings();
     }
 
-    public void load() {        
+    public void load() {  
+        KeywordSearchListsXML.getCurrent().reload();                
         reloadLists();
         reloadLangs();
         reloadEncodings();
+        tableModel.fireTableDataChanged();
     }
 
     public void store() {

nbproject/project.properties

@@ -8,6 +8,7 @@ app.version=3.0.8
 ### Build type isn't used at this point, but it may be useful
 ### Must be one of: DEVELOPMENT, RELEASE
 build.type=RELEASE
+project.org.sleuthkit.autopsy.ewfverify=ewfVerify
 #build.type=DEVELOPMENT
 update_versions=false
 #custom JVM options
@@ -32,8 +33,8 @@ modules=\
     ${project.org.sleuthkit.autopsy.sevenzip}:\
     ${project.org.sleuthkit.autopsy.scalpel}:\
     ${project.org.sleuthkit.autopsy.timeline}:\
-    ${project.org.sleuthkit.autopsy.ewfverify}:\
+    ${project.org.sleuthkit.autopsy.filetypeid}:\
-    ${project.org.sleuthkit.autopsy.filetypeid}
+    ${project.org.sleuthkit.autopsy.ewfverify}
 project.org.sleuthkit.autopsy.core=Core
 project.org.sleuthkit.autopsy.corelibs=CoreLibs
 project.org.sleuthkit.autopsy.hashdatabase=HashDatabase
@@ -46,5 +47,4 @@ project.org.sleuthkit.autopsy.sevenzip=SevenZip
 project.org.sleuthkit.autopsy.scalpel=ScalpelCarver
 project.org.sleuthkit.autopsy.timeline=Timeline
 project.org.sleuthkit.autopsy.filetypeid=FileTypeId
-project.org.sleuthkit.autopsy.ewfverify=ewfVerify