From b4d5fc1e45ff0cf4ff136b6bd7cdc5ebcebbd9f8 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 20 Feb 2014 17:46:32 -0500 Subject: [PATCH] Back up work completed on parallel file ingest --- .../ingest/GeneralIngestConfigurator.java | 167 ++++++------ .../autopsy/ingest/IngestDialogPanel.java | 244 +++++++----------- .../autopsy/ingest/IngestManager.java | 4 + .../autopsy/ingest/IngestModuleLoader.java | 11 +- .../autopsy/ingest/IngestModuleTemplate.java | 61 +++++ 5 files changed, 252 insertions(+), 235 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java index 203e98cbe8..8ab04fdbcc 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java @@ -20,7 +20,9 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Map; import javax.swing.JPanel; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.coreutils.ModuleSettings; @@ -51,92 +53,99 @@ public class GeneralIngestConfigurator implements IngestConfigurator { return loadSettingsForContext(); } - private List loadSettingsForContext() { - List messages = new ArrayList<>(); - List allModules = IngestManager.getDefault().enumerateAllModules(); + private List loadSettingsForContext() { + List moduleFactories = IngestManager.getDefault().getIngestModuleFactories(); + + // Get the enabled and disabled ingest modules settings from the user's + // config file. The default settings make all ingest modules enabled. + HashSet enabledModuleNames = getModulesNamesFromSetting(ENABLED_INGEST_MODULES_KEY, moduleListToCsv(moduleFactories)); + HashSet disabledModuleNames = getModulesNamesFromSetting(DISABLED_INGEST_MODULES_KEY, ""); - // If there is no enabled ingest modules setting for this user, default to enabling all - // of the ingest modules the IngestManager has loaded. - if (ModuleSettings.settingExists(moduleContext, ENABLED_INGEST_MODULES_KEY) == false) { - String defaultSetting = moduleListToCsv(allModules); - ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, defaultSetting); - } - - String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", "); - ArrayList enabledList = new ArrayList<>(Arrays.asList(enabledModuleNames)); - - // Check for modules that are missing from the config file - - String[] disabledModuleNames = null; - // Older config files won't have the disabled list, so don't assume it exists - if (ModuleSettings.settingExists(moduleContext, DISABLED_INGEST_MODULES_KEY)) { - disabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY).split(", "); + // Set up a collection of module templates for the view. + List moduleTemplates = new ArrayList<>(); + HashSet foundModules = new HashSet<>(); + for (IngestModuleFactory moduleFactory : moduleFactories) { + String moduleName = moduleFactory.getModuleDisplayName(); + IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, null, enabledModuleNames.contains(moduleName)); + if (!enabledModuleNames.contains(moduleName) && !enabledModuleNames.contains(moduleName)) { + // The module factory was loaded, but the module name does not + // appear in the enabled/disabled module settings. Treat the + // module as a new module and enable it by default. + moduleTemplate.setEnabled(true); + enabledModuleNames.add(moduleName); + } + foundModules.add(moduleName); } - for (IngestModuleAbstract module : allModules) { - boolean found = false; + // Check for missing modules and update the enabled/disabled ingest + // module settings. This way the settings will be up to date, even if + // save() is never called. + List errorMessages = new ArrayList<>(); + for (String moduleName : enabledModuleNames) { + if (!foundModules.contains(moduleName)) { + errorMessages.add(moduleName + " was previously enabled, but could not be found"); + enabledModuleNames.remove(moduleName); + disabledModuleNames.add(moduleName); + } + } + ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(enabledModuleNames)); + ModuleSettings.setConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(disabledModuleNames)); - // Check enabled first - for (String moduleName : enabledModuleNames) { - if (module.getName().equals(moduleName)) { - found = true; - break; - } - } - - // Then check disabled - if (!found && (disabledModuleNames != null)) { - for (String moduleName : disabledModuleNames) { - if (module.getName().equals(moduleName)) { - found = true; - break; - } - } - } - - if (!found) { - enabledList.add(module.getName()); - // It will get saved to file later - } - } - - // Get the enabled ingest modules setting, check for missing modules, and pass the setting to - // the UI component. - List enabledModules = new ArrayList<>(); - for (String moduleName : enabledList) { - if (moduleName.equals("Thunderbird Parser") - || moduleName.equals("MBox Parser")) { - moduleName = "Email Parser"; - } - - IngestModuleAbstract moduleFound = null; - for (IngestModuleAbstract module : allModules) { - if (moduleName.equals(module.getName())) { - moduleFound = module; - break; - } - } - if (moduleFound != null) { - enabledModules.add(moduleFound); - } - else { - messages.add(moduleName + " was previously enabled, but could not be found"); - } - } - ingestDialogPanel.setEnabledIngestModules(enabledModules); - - // If there is no process unallocated space flag setting, default it to false. + // Get the process unallocated space flag setting. If the setting does + // not exist yet, default it to false. if (ModuleSettings.settingExists(moduleContext, PARSE_UNALLOC_SPACE_KEY) == false) { ModuleSettings.setConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY, "false"); - } - - // Get the process unallocated space flag setting and pass it to the UI component. + } boolean processUnalloc = Boolean.parseBoolean(ModuleSettings.getConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY)); + + // Pass the settings to the nigest dialog panel. + ingestDialogPanel.setEnabledIngestModules(enabledModules); ingestDialogPanel.setProcessUnallocSpaceEnabled(processUnalloc); - return messages; + return errorMessages; } - + + private HashSet getModulesNamesFromSetting(String key, String defaultSetting) { + // Get the ingest modules setting from the user's config file. + // If there is no such setting yet, create the default setting. + if (ModuleSettings.settingExists(moduleContext, key) == false) { + ModuleSettings.setConfigSetting(moduleContext, key, defaultSetting); + } + HashSet moduleNames = new HashSet<>(); + String modulesSetting = ModuleSettings.getConfigSetting(moduleContext, key); + if (!modulesSetting.isEmpty()) { + String[] settingNames = modulesSetting.split(", "); + for (String name : settingNames) { + // Map some old core module names to the current core module names. + if (name.equals("Thunderbird Parser") || name.equals("MBox Parser")) { + moduleNames.add("Email Parser"); + } + else if (name.equals("File Extension Mismatch Detection") || name.equals("Extension Mismatch Detector")) { + moduleNames.add("File Extension Mismatch Detector"); + } + else { + moduleNames.add(name); + } + } + } + return moduleNames; + } + + private static String makeCommaSeparatedList(HashSet input) { + if (input == null || input.isEmpty()) { + return ""; + } + + ArrayList list = new ArrayList<>(); + list.addAll(input); + StringBuilder csvList = new StringBuilder(); + for (int i = 0; i < list.size() - 1; ++i) { + csvList.append(list.get(i)).append(", "); + } + csvList.append(list.get(list.size() - 1)); + return csvList.toString(); + } + @Override public JPanel getIngestConfigPanel() { // Note that this panel allows for selecting modules for the ingest process, @@ -166,18 +175,18 @@ public class GeneralIngestConfigurator implements IngestConfigurator { } } - private static String moduleListToCsv(List lst) { + private static String moduleListToCsv(List lst) { if (lst == null || lst.isEmpty()) { return ""; } StringBuilder sb = new StringBuilder(); for (int i = 0; i < lst.size() - 1; ++i) { - sb.append(lst.get(i).getName()).append(", "); + sb.append(lst.get(i).getModuleDisplayName()).append(", "); } // and the last one - sb.append(lst.get(lst.size() - 1).getName()); + sb.append(lst.get(lst.size() - 1).getModuleDisplayName()); return sb.toString(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java index ff6ef874e9..b810e51fff 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java @@ -23,10 +23,12 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; +import java.io.Serializable; import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; import java.util.Map; +import javax.swing.JPanel; import javax.swing.JTable; import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; @@ -41,92 +43,64 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; * main configuration panel for all ingest modules, reusable JPanel component */ class IngestDialogPanel extends javax.swing.JPanel { + private List moduleModels = null; + private IngestModuleModel selectedModuleModel = null; + private boolean processUnallocatedSpace = false; + private ModulesTableModel tableModel = null; - private IngestModuleAbstract currentModule; - private ModulesTableModel tableModel; - private String context; - - /** - * Creates new form IngestDialogPanel - */ - public IngestDialogPanel() { - tableModel = new ModulesTableModel(); - context = ModuleSettings.DEFAULT_CONTEXT; + IngestDialogPanel(List moduleModels, boolean processUnallocatedSpace) { + this.moduleModels = moduleModels; + this.processUnallocatedSpace = processUnallocatedSpace; initComponents(); customizeComponents(); } - - public void setContext(String context) { - this.context = context; - } - - - public IngestModuleAbstract getCurrentIngestModule() { - return currentModule; - } - - public List getModulesToStart() { - return tableModel.getSelectedModules(); - } - - public List getDisabledModules() { - return tableModel.getUnSelectedModules(); - } - - public boolean processUnallocSpaceEnabled() { + + boolean getProcessUnallocSpace() { return processUnallocCheckbox.isSelected(); } - + private void customizeComponents() { modulesTable.setModel(tableModel); modulesTable.setTableHeader(null); modulesTable.setSelectionMode(ListSelectionModel.SINGLE_SELECTION); - //custom renderer for tooltips - ModulesTableRenderer renderer = new ModulesTableRenderer(); - - //customize column witdhs - final int width = modulesScrollPane.getPreferredSize().width; - TableColumn column = null; - for (int i = 0; i < modulesTable.getColumnCount(); i++) { - column = modulesTable.getColumnModel().getColumn(i); - if (i == 0) { - column.setPreferredWidth(((int) (width * 0.15))); + // Set the column widths in the table model and add a custom cell + // renderer that will display module descriptions from the module models + // as tooltips. + ModulesTableRenderer renderer = new ModulesTableRenderer(); + int width = modulesScrollPane.getPreferredSize().width; + for (int i = 0; i < modulesTable.getColumnCount(); ++i) { + TableColumn column = modulesTable.getColumnModel().getColumn(i); + if (0 == i) { + column.setPreferredWidth(((int)(width * 0.15))); } else { column.setCellRenderer(renderer); - column.setPreferredWidth(((int) (width * 0.84))); + column.setPreferredWidth(((int)(width * 0.84))); } } + // Add a selection listener to the table model that will display the + // ingest options panel of the currently selected module model and + // enable or disable the global options panel invocation button. modulesTable.getSelectionModel().addListSelectionListener(new ListSelectionListener() { @Override public void valueChanged(ListSelectionEvent e) { - ListSelectionModel listSelectionModel = (ListSelectionModel) e.getSource(); + ListSelectionModel listSelectionModel = (ListSelectionModel)e.getSource(); if (!listSelectionModel.isSelectionEmpty()) { int index = listSelectionModel.getMinSelectionIndex(); - currentModule = tableModel.getModule(index); - - // add the module-specific configuration panel, if there is one + selectedModuleModel = moduleModels.get(index); simplePanel.removeAll(); - if (currentModule.hasSimpleConfiguration()) { - simplePanel.add(currentModule.getSimpleConfiguration(context)); + if (null != selectedModuleModel.getIngestOptionsPanel()) { + simplePanel.add(selectedModuleModel.getIngestOptionsPanel()); } simplePanel.revalidate(); simplePanel.repaint(); - advancedButton.setEnabled(currentModule.hasAdvancedConfiguration()); - } else { - currentModule = null; + advancedButton.setEnabled(null != selectedModuleModel.getGlobalOptionsPanel()); } } }); - } - - public void setProcessUnallocSpaceEnabled(final boolean enabled) { - processUnallocCheckbox.setSelected(enabled); - } - - public void setEnabledIngestModules(List enabledModules) { - tableModel.setSelectedModules(enabledModules); + + processUnallocCheckbox.setSelected(processUnallocatedSpace); } /** @@ -277,11 +251,11 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; dialog.close(); } }); - dialog.display(currentModule.getAdvancedConfiguration(context)); + dialog.display(selectedModuleModel.getGlobalOptionsPanel()); }//GEN-LAST:event_advancedButtonActionPerformed private void processUnallocCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_processUnallocCheckboxActionPerformed - // nothing to do here + processUnallocatedSpace = processUnallocCheckbox.isSelected(); }//GEN-LAST:event_processUnallocCheckboxActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton advancedButton; @@ -296,20 +270,57 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; private javax.swing.ButtonGroup timeGroup; // End of variables declaration//GEN-END:variables - private class ModulesTableModel extends AbstractTableModel { - - private List>moduleData = new ArrayList<>(); + static class IngestModuleModel { + private final IngestModuleFactory moduleFactory; + private final JPanel ingestOptionsPanel; + private final JPanel globalOptionsPanel; + private boolean enabled = true; - public ModulesTableModel() { - List modules = IngestManager.getDefault().enumerateAllModules(); - for (IngestModuleAbstract ingestModuleAbstract : modules) { - moduleData.add(new AbstractMap.SimpleEntry<>(ingestModuleAbstract, Boolean.TRUE)); + IngestModuleModel(IngestModuleFactory moduleFactory, Serializable ingestOptions, boolean enabled) { + this.moduleFactory = moduleFactory; + this.enabled = enabled; + if (moduleFactory.providesIngestOptionsPanels()) { + ingestOptionsPanel = moduleFactory.getIngestOptionsPanel(ingestOptions); + } + else { + ingestOptionsPanel = null; + } + if (moduleFactory.providesGlobalOptionsPanels()) { + + } + else { } } + + String getModuleDisplayName() { + return moduleFactory.getModuleDisplayName(); + } + String getModuleDescription() { + return moduleFactory.getModuleDescription(); + } + + void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + boolean isEnabled() { + return enabled; + } + + JPanel getIngestOptionsPanel() { + return ingestOptionsPanel; + } + + JPanel getGlobalOptionsPanel() { + return globalOptionsPanel; + } + } + + private class ModulesTableModel extends AbstractTableModel { @Override public int getRowCount() { - return moduleData.size(); + return moduleModels.size(); } @Override @@ -319,11 +330,12 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; @Override public Object getValueAt(int rowIndex, int columnIndex) { - Map.Entry entry = moduleData.get(rowIndex); + IngestModuleTemplate moduleTemplate = moduleModels.get(rowIndex); if (columnIndex == 0) { - return entry.getValue(); - } else { - return entry.getKey().getName(); + return moduleTemplate.isEnabled(); + } + else { + return moduleTemplate.getModuleDisplayName(); } } @@ -335,87 +347,14 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; @Override public void setValueAt(Object aValue, int rowIndex, int columnIndex) { if (columnIndex == 0) { - moduleData.get(rowIndex).setValue((Boolean)aValue); + moduleModels.get(rowIndex).setEnabled((boolean)aValue); } } @Override public Class getColumnClass(int c) { return getValueAt(0, c).getClass(); - } - - public List getSelectedModules() { - List selectedModules = new ArrayList<>(); - for (Map.Entry entry : moduleData) { - if (entry.getValue().booleanValue()) { - selectedModules.add(entry.getKey()); - } - } - return selectedModules; - } - - public List getUnSelectedModules() { - List unselectedModules = new ArrayList<>(); - for (Map.Entry entry : moduleData) { - if (!entry.getValue().booleanValue()) { - unselectedModules.add(entry.getKey()); - } - } - return unselectedModules; } - - /** - * Sets the given modules as selected in the modules table - * @param selectedModules - */ - public void setSelectedModules(List selectedModules) { - // unselect all modules - for (Map.Entry entry : moduleData) { - entry.setValue(Boolean.FALSE); - } - - // select only the given modules - for (IngestModuleAbstract selectedModule : selectedModules) { - getEntryForModule(selectedModule).setValue(Boolean.TRUE); - } - - // tell everyone about it - fireTableDataChanged(); - } - - /** - * Sets the given modules as NOT selected in the modules table - * @param selectedModules - */ - public void setUnselectedModules(List unselectedModules) { - // select all modules - for (Map.Entry entry : moduleData) { - entry.setValue(Boolean.TRUE); - } - - // unselect only the given modules - for (IngestModuleAbstract unselectedModule : unselectedModules) { - getEntryForModule(unselectedModule).setValue(Boolean.FALSE); - } - - // tell everyone about it - fireTableDataChanged(); - } - - public IngestModuleAbstract getModule(int row) { - return moduleData.get(row).getKey(); - } - - private Map.Entry getEntryForModule(IngestModuleAbstract module) { - Map.Entry entry = null; - for (Map.Entry anEntry : moduleData) { - if (anEntry.getKey().equals(module)) { - entry = anEntry; - break; - } - } - return entry; - } } /** @@ -426,20 +365,15 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; List tooltips = new ArrayList<>(); public ModulesTableRenderer() { - List modules = IngestManager.getDefault().enumerateAllModules(); - for (IngestModuleAbstract ingestModuleAbstract : modules) { - tooltips.add(ingestModuleAbstract.getDescription()); + for (IngestModuleTemplate moduleTemplate : moduleModels) { + tooltips.add(moduleTemplate.getModuleDescription()); } } @Override - public Component getTableCellRendererComponent( - JTable table, Object value, - boolean isSelected, boolean hasFocus, - int row, int column) { + public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column); - - if (column == 1) { + if (1 == column) { setToolTipText(tooltips.get(row)); } return this; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 6e079a2d53..8c4250901b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -792,6 +792,10 @@ public class IngestManager { modules.addAll(enumerateAbstractFileModules()); return modules; } + + List getIngestModuleFactories() { + return moduleLoader.getIngestModuleFactories(); + } //data source worker to remove itself when complete or interrupted void removeDataSourceIngestWorker(IngestDataSourceThread worker) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java index c3f406c89e..c3ea5433a2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java @@ -91,7 +91,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; * code refactored */ final class IngestModuleLoader { - + private ArrayList moduleFactories = new ArrayList<>(); private static final String PIPELINE_CONFIG_XML = "pipeline_config.xml"; private static final String XSDFILE = "PipelineConfigSchema.xsd"; private String absFilePath; @@ -456,6 +456,10 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; return urls; } + List getIngestModuleFactories() { + return moduleFactories; + } + /** * Auto-discover ingest modules in all platform modules that are "enabled" * If discovered ingest module is not already in XML config, add it do @@ -466,6 +470,11 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; @SuppressWarnings("unchecked") private void autodiscover() throws IngestModuleLoaderException { + Collection factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class); + moduleFactories.addAll(factories); + +// moduleFactories + // Use Lookup to find the other NBM modules. We'll later search them for ingest modules Collection moduleInfos = Lookup.getDefault().lookupAll(ModuleInfo.class); logger.log(Level.INFO, "Autodiscovery, found #platform modules: " + moduleInfos.size()); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java new file mode 100755 index 0000000000..5580dfa171 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java @@ -0,0 +1,61 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.ingest; + +import java.io.Serializable; + +/** + * RJCTODO + */ +public class IngestModuleTemplate { + private final IngestModuleFactory moduleFactory; + private Serializable ingestOptions = null; + private boolean enabled = true; + + IngestModuleTemplate(IngestModuleFactory moduleFactory, Serializable ingestOptions, boolean enabled) { + this.moduleFactory = moduleFactory; + this.ingestOptions = ingestOptions; + this.enabled = enabled; + } + + String getModuleDisplayName() { + return moduleFactory.getModuleDisplayName(); + } + + String getModuleDescription() { + return moduleFactory.getModuleDescription(); + } + + Serializable getIngestOptions() { + return ingestOptions; + } + + void setIngestOptions(Serializable ingestOptions) { + this.ingestOptions = ingestOptions; + } + + void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + boolean isEnabled() { + return enabled; + } +}