diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 3b77c81a7e..6166fdd4d6 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -126,6 +126,36 @@ public class BlackboardArtifactNode extends DisplayableItemNode { } final int artifactTypeId = artifact.getArtifactTypeID(); + // If mismatch, add props for extension and file type + if (artifactTypeId == BlackboardArtifact.ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED.getTypeID()) { + String actualExt = ""; + int i = associated.getName().lastIndexOf("."); + if ((i > -1) && ((i + 1) < associated.getName().length())) { + actualExt = associated.getName().substring(i + 1).toLowerCase(); + } + ss.put(new NodeProperty("Extension", "Extension", NO_DESCR, actualExt)); + + try { + String actualMimeType = ""; + ArrayList artList = associated.getAllArtifacts(); + for (BlackboardArtifact art : artList) { + List atrList = art.getAttributes(); + for (BlackboardAttribute att : atrList) { + if (att.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID()) { + actualMimeType = att.getValueString(); + } + } + } + if (actualMimeType.isEmpty()) { + logger.log(Level.WARNING, "Could not find expected TSK_FILE_TYPE_SIG attribute."); + } else { + ss.put(new NodeProperty("MIME Type", "MIME Type", NO_DESCR, actualMimeType)); + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Error while searching for TSK_FILE_TYPE_SIG attribute: ", ex); + } + } + if (Arrays.asList(SHOW_UNIQUE_PATH).contains(artifactTypeId)) { String sourcePath = ""; try { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java index eaeb0eefc6..a9769a5b5b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.swing.JPanel; import org.openide.util.lookup.ServiceProvider; @@ -29,6 +30,7 @@ import org.sleuthkit.datamodel.Content; public class GeneralIngestConfigurator implements IngestConfigurator { public static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; + public static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; public static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; private List contentToIngest; private IngestManager manager; @@ -51,20 +53,57 @@ public class GeneralIngestConfigurator implements IngestConfigurator { private List loadSettingsForContext() { List messages = new ArrayList<>(); + List allModules = IngestManager.getDefault().enumerateAllModules(); // If there is no enabled ingest modules setting for this user, default to enabling all // of the ingest modules the IngestManager has loaded. if (ModuleSettings.settingExists(moduleContext, ENABLED_INGEST_MODULES_KEY) == false) { - String defaultSetting = moduleListToCsv(IngestManager.getDefault().enumerateAllModules()); + String defaultSetting = moduleListToCsv(allModules); ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, defaultSetting); } + String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", "); + ArrayList enabledList = new ArrayList<>(Arrays.asList(enabledModuleNames)); + + // Check for modules that are missing from the config file + + String[] disabledModuleNames = null; + // Older config files won't have the disabled list, so don't assume it exists + if (ModuleSettings.settingExists(moduleContext, DISABLED_INGEST_MODULES_KEY)) { + disabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY).split(", "); + } + + for (IngestModuleAbstract module : allModules) { + boolean found = false; + + // Check enabled first + for (String moduleName : enabledModuleNames) { + if (module.getName().equals(moduleName)) { + found = true; + break; + } + } + + // Then check disabled + if (!found && (disabledModuleNames != null)) { + for (String moduleName : disabledModuleNames) { + if (module.getName().equals(moduleName)) { + found = true; + break; + } + } + } + + if (!found) { + enabledList.add(module.getName()); + // It will get saved to file later + } + } + // Get the enabled ingest modules setting, check for missing modules, and pass the setting to // the UI component. - List allModules = IngestManager.getDefault().enumerateAllModules(); - String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", "); List enabledModules = new ArrayList<>(); - for (String moduleName : enabledModuleNames) { + for (String moduleName : enabledList) { if (moduleName.equals("Thunderbird Parser") || moduleName.equals("MBox Parser")) { moduleName = "Email Parser"; @@ -112,6 +151,10 @@ public class GeneralIngestConfigurator implements IngestConfigurator { String enabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getModulesToStart()); ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, enabledModulesCsvList); + // Save the user's configuration of the set of disabled ingest modules. + String disabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getDisabledModules()); + ModuleSettings.setConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY, disabledModulesCsvList); + // Save the user's setting for the process unallocated space flag. String processUnalloc = Boolean.toString(ingestDialogPanel.processUnallocSpaceEnabled()); ModuleSettings.setConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY, processUnalloc); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java index 331c727576..198f19da13 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java @@ -60,7 +60,7 @@ public class IngestDialogPanel extends javax.swing.JPanel { this.context = context; } - + public IngestModuleAbstract getCurrentIngestModule() { return currentModule; } @@ -69,6 +69,10 @@ public class IngestDialogPanel extends javax.swing.JPanel { return tableModel.getSelectedModules(); } + public List getDisabledModules() { + return tableModel.getUnSelectedModules(); + } + public boolean processUnallocSpaceEnabled() { return processUnallocCheckbox.isSelected(); } @@ -350,6 +354,16 @@ public class IngestDialogPanel extends javax.swing.JPanel { return selectedModules; } + public List getUnSelectedModules() { + List unselectedModules = new ArrayList<>(); + for (Map.Entry entry : moduleData) { + if (!entry.getValue().booleanValue()) { + unselectedModules.add(entry.getKey()); + } + } + return unselectedModules; + } + /** * Sets the given modules as selected in the modules table * @param selectedModules diff --git a/FileTypeId/nbproject/project.xml b/FileTypeId/nbproject/project.xml index bcf4cb3009..8d3ea957d3 100644 --- a/FileTypeId/nbproject/project.xml +++ b/FileTypeId/nbproject/project.xml @@ -91,7 +91,9 @@ - + + org.sleuthkit.autopsy.filetypeid + ext/tika-core-1.2.jar release/modules/ext/tika-core-1.2.jar diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java index a05091c139..47f4feb77b 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java @@ -36,5 +36,7 @@ public interface FileTypeDetectionInterface { } // You only have one job - FileIdInfo attemptMatch(AbstractFile abstractFile); + FileIdInfo attemptMatch(AbstractFile abstractFile); + + boolean isMimeTypeDetectable(String mimeType); } diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java index 81c623d9b3..39878ca1d6 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java @@ -48,7 +48,7 @@ public class FileTypeIdIngestModule extends org.sleuthkit.autopsy.ingest.IngestM private static long matchTime = 0; private static int messageId = 0; private static long numFiles = 0; - private static boolean skipKnown = false; + private static boolean skipKnown = true; private FileTypeIdSimpleConfigPanel simpleConfigPanel; private IngestServices services; @@ -173,4 +173,15 @@ public class FileTypeIdIngestModule extends org.sleuthkit.autopsy.ingest.IngestM public static void setSkipKnown(boolean flag) { skipKnown = flag; } + + /** + * Validate if a given mime type is in the detector's registry. + * @param mimeType Full string of mime type, e.g. "text/html" + * @return true if detectable + */ + public static boolean isMimeTypeDetectable(String mimeType) { + FileTypeDetectionInterface detector = new TikaFileTypeDetector(); + return detector.isMimeTypeDetectable(mimeType); + } + } \ No newline at end of file diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form index fe15876df1..29004307b5 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form @@ -36,6 +36,7 @@ + diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java index 5a3f11eda9..43576d8184 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java @@ -44,6 +44,7 @@ public class FileTypeIdSimpleConfigPanel extends javax.swing.JPanel { skipKnownCheckBox = new javax.swing.JCheckBox(); + skipKnownCheckBox.setSelected(true); skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text")); // NOI18N skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.toolTipText")); // NOI18N skipKnownCheckBox.addActionListener(new java.awt.event.ActionListener() { diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java index 365584db7f..ff24f485e5 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java @@ -18,8 +18,11 @@ */ package org.sleuthkit.autopsy.filetypeid; +import java.util.SortedSet; import org.openide.util.Exceptions; import org.apache.tika.Tika; +import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.MimeTypes; import org.sleuthkit.datamodel.AbstractFile; @@ -52,4 +55,27 @@ public class TikaFileTypeDetector implements FileTypeDetectionInterface { } } + /** + * Validate if a given mime type is in the registry. + * For Tika, we remove the string "tika" from all MIME names, + * e.g. use "application/x-msoffice" NOT "application/x-tika-msoffice" + * @param mimeType Full string of mime type, e.g. "text/html" + * @return true if detectable + */ + @Override + public boolean isMimeTypeDetectable(String mimeType) { + boolean ret = false; + + SortedSet m = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); + String[] split = mimeType.split("/"); + + if (split.length == 2) { + String type = split[0]; + String subtype = split[1]; + MediaType mediaType = new MediaType(type, subtype); + ret = m.contains(mediaType); + } + + return ret; + } } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index f3f63ebd16..6e8d895f06 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -50,6 +50,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { public final static String MODULE_DESCRIPTION = "Identifies known and notables files using supplied hash databases, such as a standard NSRL database."; final public static String MODULE_VERSION = Version.getVersion(); private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName()); + private static final int MAX_COMMENT_SIZE = 500; private HashDbSimpleConfigPanel simpleConfigPanel; private HashDbConfigPanel advancedConfigPanel; private IngestServices services; @@ -227,9 +228,24 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { services.postMessage(IngestMessage.createErrorMessage(++messageId, HashDbIngestModule.this, "Hash Lookup Error: " + name, "Error encountered while setting known bad state for " + name + ".")); ret = ProcessResult.ERROR; - } + } String hashSetName = db.getHashSetName(); - postHashSetHitToBlackboard(file, md5Hash, hashSetName, db.getSendIngestMessages()); + + String comment = ""; + ArrayList comments = db.lookUp(file).getComments(); + int i = 0; + for (String c : comments) { + comment += c; + if (++i > 1) { + c += ". "; + } + if (comment.length() > MAX_COMMENT_SIZE) { + comment = comment.substring(0, MAX_COMMENT_SIZE) + "..."; + break; + } + } + + postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages()); } lookuptime += (System.currentTimeMillis() - lookupstart); } catch (TskException ex) { @@ -271,7 +287,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { return ret; } - private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, boolean showInboxMessage) { + private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) { try { BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT); //TODO Revisit usage of deprecated constructor as per TSK-583 @@ -280,6 +296,9 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { badFile.addAttribute(att2); BlackboardAttribute att3 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5.getTypeID(), MODULE_NAME, md5Hash); badFile.addAttribute(att3); + BlackboardAttribute att4 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT.getTypeID(), MODULE_NAME, comment); + badFile.addAttribute(att4); + if (showInboxMessage) { StringBuilder detailsSb = new StringBuilder(); //details diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 592e363c28..5978c64bef 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,6 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.io.IOException; import java.io.InputStream; -import java.lang.Long; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -128,7 +127,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { private static List textExtractors; private static AbstractFileStringExtract stringExtractor; private boolean initialized = false; - private KeywordSearchConfigurationPanel panel; + private KeywordSearchIngestSimplePanel simpleConfigPanel; + private KeywordSearchConfigurationPanel advancedConfigPanel; private Tika tikaFormatDetector; @@ -436,26 +436,36 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { @Override public javax.swing.JPanel getSimpleConfiguration(String context) { KeywordSearchListsXML.getCurrent().reload(); - return new KeywordSearchIngestSimplePanel(); + + if (null == simpleConfigPanel) { + simpleConfigPanel = new KeywordSearchIngestSimplePanel(); + } + else { + simpleConfigPanel.load(); + } + + return simpleConfigPanel; } @Override public javax.swing.JPanel getAdvancedConfiguration(String context) { - //return KeywordSearchConfigurationPanel.getDefault(); - getPanel().load(); - return getPanel(); - } - - private KeywordSearchConfigurationPanel getPanel() { - if (panel == null) { - panel = new KeywordSearchConfigurationPanel(); + if (advancedConfigPanel == null) { + advancedConfigPanel = new KeywordSearchConfigurationPanel(); } - return panel; + + advancedConfigPanel.load(); + return advancedConfigPanel; } @Override public void saveAdvancedConfiguration() { - getPanel().store(); + if (advancedConfigPanel != null) { + advancedConfigPanel.store(); + } + + if (simpleConfigPanel != null) { + simpleConfigPanel.load(); + } } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java index fb1bc964a8..7c6b8090d6 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java @@ -70,10 +70,12 @@ public class KeywordSearchIngestSimplePanel extends javax.swing.JPanel { reloadEncodings(); } - public void load() { + public void load() { + KeywordSearchListsXML.getCurrent().reload(); reloadLists(); reloadLangs(); reloadEncodings(); + tableModel.fireTableDataChanged(); } public void store() { diff --git a/nbproject/project.properties b/nbproject/project.properties index 5176c4e932..aea6502156 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -8,6 +8,7 @@ app.version=3.0.8 ### Build type isn't used at this point, but it may be useful ### Must be one of: DEVELOPMENT, RELEASE build.type=RELEASE +project.org.sleuthkit.autopsy.ewfverify=ewfVerify #build.type=DEVELOPMENT update_versions=false #custom JVM options @@ -32,8 +33,8 @@ modules=\ ${project.org.sleuthkit.autopsy.sevenzip}:\ ${project.org.sleuthkit.autopsy.scalpel}:\ ${project.org.sleuthkit.autopsy.timeline}:\ - ${project.org.sleuthkit.autopsy.ewfverify}:\ - ${project.org.sleuthkit.autopsy.filetypeid} + ${project.org.sleuthkit.autopsy.filetypeid}:\ + ${project.org.sleuthkit.autopsy.ewfverify} project.org.sleuthkit.autopsy.core=Core project.org.sleuthkit.autopsy.corelibs=CoreLibs project.org.sleuthkit.autopsy.hashdatabase=HashDatabase @@ -46,5 +47,4 @@ project.org.sleuthkit.autopsy.sevenzip=SevenZip project.org.sleuthkit.autopsy.scalpel=ScalpelCarver project.org.sleuthkit.autopsy.timeline=Timeline project.org.sleuthkit.autopsy.filetypeid=FileTypeId -project.org.sleuthkit.autopsy.ewfverify=ewfVerify