Back up work completed on parallel file ingest

This commit is contained in:
Richard Cordovano 2014-02-20 17:46:32 -05:00
parent fef91bca20
commit b4d5fc1e45
5 changed files with 252 additions and 235 deletions

View File

@ -20,7 +20,9 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import javax.swing.JPanel;
import org.openide.util.lookup.ServiceProvider;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
@ -51,92 +53,99 @@ public class GeneralIngestConfigurator implements IngestConfigurator {
return loadSettingsForContext();
}
private List<String> loadSettingsForContext() {
List<String> messages = new ArrayList<>();
List<IngestModuleAbstract> allModules = IngestManager.getDefault().enumerateAllModules();
private List<String> loadSettingsForContext() {
List<IngestModuleFactory> moduleFactories = IngestManager.getDefault().getIngestModuleFactories();
// Get the enabled and disabled ingest modules settings from the user's
// config file. The default settings make all ingest modules enabled.
HashSet<String> enabledModuleNames = getModulesNamesFromSetting(ENABLED_INGEST_MODULES_KEY, moduleListToCsv(moduleFactories));
HashSet<String> disabledModuleNames = getModulesNamesFromSetting(DISABLED_INGEST_MODULES_KEY, "");
// If there is no enabled ingest modules setting for this user, default to enabling all
// of the ingest modules the IngestManager has loaded.
if (ModuleSettings.settingExists(moduleContext, ENABLED_INGEST_MODULES_KEY) == false) {
String defaultSetting = moduleListToCsv(allModules);
ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, defaultSetting);
}
String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", ");
ArrayList<String> enabledList = new ArrayList<>(Arrays.asList(enabledModuleNames));
// Check for modules that are missing from the config file
String[] disabledModuleNames = null;
// Older config files won't have the disabled list, so don't assume it exists
if (ModuleSettings.settingExists(moduleContext, DISABLED_INGEST_MODULES_KEY)) {
disabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY).split(", ");
// Set up a collection of module templates for the view.
List<IngestModuleTemplate> moduleTemplates = new ArrayList<>();
HashSet<String> foundModules = new HashSet<>();
for (IngestModuleFactory moduleFactory : moduleFactories) {
String moduleName = moduleFactory.getModuleDisplayName();
IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, null, enabledModuleNames.contains(moduleName));
if (!enabledModuleNames.contains(moduleName) && !enabledModuleNames.contains(moduleName)) {
// The module factory was loaded, but the module name does not
// appear in the enabled/disabled module settings. Treat the
// module as a new module and enable it by default.
moduleTemplate.setEnabled(true);
enabledModuleNames.add(moduleName);
}
foundModules.add(moduleName);
}
for (IngestModuleAbstract module : allModules) {
boolean found = false;
// Check for missing modules and update the enabled/disabled ingest
// module settings. This way the settings will be up to date, even if
// save() is never called.
List<String> errorMessages = new ArrayList<>();
for (String moduleName : enabledModuleNames) {
if (!foundModules.contains(moduleName)) {
errorMessages.add(moduleName + " was previously enabled, but could not be found");
enabledModuleNames.remove(moduleName);
disabledModuleNames.add(moduleName);
}
}
ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(enabledModuleNames));
ModuleSettings.setConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(disabledModuleNames));
// Check enabled first
for (String moduleName : enabledModuleNames) {
if (module.getName().equals(moduleName)) {
found = true;
break;
}
}
// Then check disabled
if (!found && (disabledModuleNames != null)) {
for (String moduleName : disabledModuleNames) {
if (module.getName().equals(moduleName)) {
found = true;
break;
}
}
}
if (!found) {
enabledList.add(module.getName());
// It will get saved to file later
}
}
// Get the enabled ingest modules setting, check for missing modules, and pass the setting to
// the UI component.
List<IngestModuleAbstract> enabledModules = new ArrayList<>();
for (String moduleName : enabledList) {
if (moduleName.equals("Thunderbird Parser")
|| moduleName.equals("MBox Parser")) {
moduleName = "Email Parser";
}
IngestModuleAbstract moduleFound = null;
for (IngestModuleAbstract module : allModules) {
if (moduleName.equals(module.getName())) {
moduleFound = module;
break;
}
}
if (moduleFound != null) {
enabledModules.add(moduleFound);
}
else {
messages.add(moduleName + " was previously enabled, but could not be found");
}
}
ingestDialogPanel.setEnabledIngestModules(enabledModules);
// If there is no process unallocated space flag setting, default it to false.
// Get the process unallocated space flag setting. If the setting does
// not exist yet, default it to false.
if (ModuleSettings.settingExists(moduleContext, PARSE_UNALLOC_SPACE_KEY) == false) {
ModuleSettings.setConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY, "false");
}
// Get the process unallocated space flag setting and pass it to the UI component.
}
boolean processUnalloc = Boolean.parseBoolean(ModuleSettings.getConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY));
// Pass the settings to the nigest dialog panel.
ingestDialogPanel.setEnabledIngestModules(enabledModules);
ingestDialogPanel.setProcessUnallocSpaceEnabled(processUnalloc);
return messages;
return errorMessages;
}
private HashSet<String> getModulesNamesFromSetting(String key, String defaultSetting) {
// Get the ingest modules setting from the user's config file.
// If there is no such setting yet, create the default setting.
if (ModuleSettings.settingExists(moduleContext, key) == false) {
ModuleSettings.setConfigSetting(moduleContext, key, defaultSetting);
}
HashSet<String> moduleNames = new HashSet<>();
String modulesSetting = ModuleSettings.getConfigSetting(moduleContext, key);
if (!modulesSetting.isEmpty()) {
String[] settingNames = modulesSetting.split(", ");
for (String name : settingNames) {
// Map some old core module names to the current core module names.
if (name.equals("Thunderbird Parser") || name.equals("MBox Parser")) {
moduleNames.add("Email Parser");
}
else if (name.equals("File Extension Mismatch Detection") || name.equals("Extension Mismatch Detector")) {
moduleNames.add("File Extension Mismatch Detector");
}
else {
moduleNames.add(name);
}
}
}
return moduleNames;
}
private static String makeCommaSeparatedList(HashSet<String> input) {
if (input == null || input.isEmpty()) {
return "";
}
ArrayList<String> list = new ArrayList<>();
list.addAll(input);
StringBuilder csvList = new StringBuilder();
for (int i = 0; i < list.size() - 1; ++i) {
csvList.append(list.get(i)).append(", ");
}
csvList.append(list.get(list.size() - 1));
return csvList.toString();
}
@Override
public JPanel getIngestConfigPanel() {
// Note that this panel allows for selecting modules for the ingest process,
@ -166,18 +175,18 @@ public class GeneralIngestConfigurator implements IngestConfigurator {
}
}
private static String moduleListToCsv(List<IngestModuleAbstract> lst) {
private static String moduleListToCsv(List<IngestModuleFactory> lst) {
if (lst == null || lst.isEmpty()) {
return "";
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < lst.size() - 1; ++i) {
sb.append(lst.get(i).getName()).append(", ");
sb.append(lst.get(i).getModuleDisplayName()).append(", ");
}
// and the last one
sb.append(lst.get(lst.size() - 1).getName());
sb.append(lst.get(lst.size() - 1).getModuleDisplayName());
return sb.toString();
}

View File

@ -23,10 +23,12 @@ import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.io.Serializable;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.swing.JPanel;
import javax.swing.JTable;
import javax.swing.ListSelectionModel;
import javax.swing.event.ListSelectionEvent;
@ -41,92 +43,64 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings;
* main configuration panel for all ingest modules, reusable JPanel component
*/
class IngestDialogPanel extends javax.swing.JPanel {
private List<IngestModuleModel> moduleModels = null;
private IngestModuleModel selectedModuleModel = null;
private boolean processUnallocatedSpace = false;
private ModulesTableModel tableModel = null;
private IngestModuleAbstract currentModule;
private ModulesTableModel tableModel;
private String context;
/**
* Creates new form IngestDialogPanel
*/
public IngestDialogPanel() {
tableModel = new ModulesTableModel();
context = ModuleSettings.DEFAULT_CONTEXT;
IngestDialogPanel(List<IngestModuleModel> moduleModels, boolean processUnallocatedSpace) {
this.moduleModels = moduleModels;
this.processUnallocatedSpace = processUnallocatedSpace;
initComponents();
customizeComponents();
}
public void setContext(String context) {
this.context = context;
}
public IngestModuleAbstract getCurrentIngestModule() {
return currentModule;
}
public List<IngestModuleAbstract> getModulesToStart() {
return tableModel.getSelectedModules();
}
public List<IngestModuleAbstract> getDisabledModules() {
return tableModel.getUnSelectedModules();
}
public boolean processUnallocSpaceEnabled() {
boolean getProcessUnallocSpace() {
return processUnallocCheckbox.isSelected();
}
private void customizeComponents() {
modulesTable.setModel(tableModel);
modulesTable.setTableHeader(null);
modulesTable.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
//custom renderer for tooltips
ModulesTableRenderer renderer = new ModulesTableRenderer();
//customize column witdhs
final int width = modulesScrollPane.getPreferredSize().width;
TableColumn column = null;
for (int i = 0; i < modulesTable.getColumnCount(); i++) {
column = modulesTable.getColumnModel().getColumn(i);
if (i == 0) {
column.setPreferredWidth(((int) (width * 0.15)));
// Set the column widths in the table model and add a custom cell
// renderer that will display module descriptions from the module models
// as tooltips.
ModulesTableRenderer renderer = new ModulesTableRenderer();
int width = modulesScrollPane.getPreferredSize().width;
for (int i = 0; i < modulesTable.getColumnCount(); ++i) {
TableColumn column = modulesTable.getColumnModel().getColumn(i);
if (0 == i) {
column.setPreferredWidth(((int)(width * 0.15)));
} else {
column.setCellRenderer(renderer);
column.setPreferredWidth(((int) (width * 0.84)));
column.setPreferredWidth(((int)(width * 0.84)));
}
}
// Add a selection listener to the table model that will display the
// ingest options panel of the currently selected module model and
// enable or disable the global options panel invocation button.
modulesTable.getSelectionModel().addListSelectionListener(new ListSelectionListener() {
@Override
public void valueChanged(ListSelectionEvent e) {
ListSelectionModel listSelectionModel = (ListSelectionModel) e.getSource();
ListSelectionModel listSelectionModel = (ListSelectionModel)e.getSource();
if (!listSelectionModel.isSelectionEmpty()) {
int index = listSelectionModel.getMinSelectionIndex();
currentModule = tableModel.getModule(index);
// add the module-specific configuration panel, if there is one
selectedModuleModel = moduleModels.get(index);
simplePanel.removeAll();
if (currentModule.hasSimpleConfiguration()) {
simplePanel.add(currentModule.getSimpleConfiguration(context));
if (null != selectedModuleModel.getIngestOptionsPanel()) {
simplePanel.add(selectedModuleModel.getIngestOptionsPanel());
}
simplePanel.revalidate();
simplePanel.repaint();
advancedButton.setEnabled(currentModule.hasAdvancedConfiguration());
} else {
currentModule = null;
advancedButton.setEnabled(null != selectedModuleModel.getGlobalOptionsPanel());
}
}
});
}
public void setProcessUnallocSpaceEnabled(final boolean enabled) {
processUnallocCheckbox.setSelected(enabled);
}
public void setEnabledIngestModules(List<IngestModuleAbstract> enabledModules) {
tableModel.setSelectedModules(enabledModules);
processUnallocCheckbox.setSelected(processUnallocatedSpace);
}
/**
@ -277,11 +251,11 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings;
dialog.close();
}
});
dialog.display(currentModule.getAdvancedConfiguration(context));
dialog.display(selectedModuleModel.getGlobalOptionsPanel());
}//GEN-LAST:event_advancedButtonActionPerformed
private void processUnallocCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_processUnallocCheckboxActionPerformed
// nothing to do here
processUnallocatedSpace = processUnallocCheckbox.isSelected();
}//GEN-LAST:event_processUnallocCheckboxActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton advancedButton;
@ -296,20 +270,57 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings;
private javax.swing.ButtonGroup timeGroup;
// End of variables declaration//GEN-END:variables
private class ModulesTableModel extends AbstractTableModel {
private List<Map.Entry<IngestModuleAbstract, Boolean>>moduleData = new ArrayList<>();
static class IngestModuleModel {
private final IngestModuleFactory moduleFactory;
private final JPanel ingestOptionsPanel;
private final JPanel globalOptionsPanel;
private boolean enabled = true;
public ModulesTableModel() {
List<IngestModuleAbstract> modules = IngestManager.getDefault().enumerateAllModules();
for (IngestModuleAbstract ingestModuleAbstract : modules) {
moduleData.add(new AbstractMap.SimpleEntry<>(ingestModuleAbstract, Boolean.TRUE));
IngestModuleModel(IngestModuleFactory moduleFactory, Serializable ingestOptions, boolean enabled) {
this.moduleFactory = moduleFactory;
this.enabled = enabled;
if (moduleFactory.providesIngestOptionsPanels()) {
ingestOptionsPanel = moduleFactory.getIngestOptionsPanel(ingestOptions);
}
else {
ingestOptionsPanel = null;
}
if (moduleFactory.providesGlobalOptionsPanels()) {
}
else {
}
}
String getModuleDisplayName() {
return moduleFactory.getModuleDisplayName();
}
String getModuleDescription() {
return moduleFactory.getModuleDescription();
}
void setEnabled(boolean enabled) {
this.enabled = enabled;
}
boolean isEnabled() {
return enabled;
}
JPanel getIngestOptionsPanel() {
return ingestOptionsPanel;
}
JPanel getGlobalOptionsPanel() {
return globalOptionsPanel;
}
}
private class ModulesTableModel extends AbstractTableModel {
@Override
public int getRowCount() {
return moduleData.size();
return moduleModels.size();
}
@Override
@ -319,11 +330,12 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings;
@Override
public Object getValueAt(int rowIndex, int columnIndex) {
Map.Entry<IngestModuleAbstract, Boolean> entry = moduleData.get(rowIndex);
IngestModuleTemplate moduleTemplate = moduleModels.get(rowIndex);
if (columnIndex == 0) {
return entry.getValue();
} else {
return entry.getKey().getName();
return moduleTemplate.isEnabled();
}
else {
return moduleTemplate.getModuleDisplayName();
}
}
@ -335,87 +347,14 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings;
@Override
public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
if (columnIndex == 0) {
moduleData.get(rowIndex).setValue((Boolean)aValue);
moduleModels.get(rowIndex).setEnabled((boolean)aValue);
}
}
@Override
public Class<?> getColumnClass(int c) {
return getValueAt(0, c).getClass();
}
public List<IngestModuleAbstract> getSelectedModules() {
List<IngestModuleAbstract> selectedModules = new ArrayList<>();
for (Map.Entry<IngestModuleAbstract, Boolean> entry : moduleData) {
if (entry.getValue().booleanValue()) {
selectedModules.add(entry.getKey());
}
}
return selectedModules;
}
public List<IngestModuleAbstract> getUnSelectedModules() {
List<IngestModuleAbstract> unselectedModules = new ArrayList<>();
for (Map.Entry<IngestModuleAbstract, Boolean> entry : moduleData) {
if (!entry.getValue().booleanValue()) {
unselectedModules.add(entry.getKey());
}
}
return unselectedModules;
}
/**
* Sets the given modules as selected in the modules table
* @param selectedModules
*/
public void setSelectedModules(List<IngestModuleAbstract> selectedModules) {
// unselect all modules
for (Map.Entry<IngestModuleAbstract, Boolean> entry : moduleData) {
entry.setValue(Boolean.FALSE);
}
// select only the given modules
for (IngestModuleAbstract selectedModule : selectedModules) {
getEntryForModule(selectedModule).setValue(Boolean.TRUE);
}
// tell everyone about it
fireTableDataChanged();
}
/**
* Sets the given modules as NOT selected in the modules table
* @param selectedModules
*/
public void setUnselectedModules(List<IngestModuleAbstract> unselectedModules) {
// select all modules
for (Map.Entry<IngestModuleAbstract, Boolean> entry : moduleData) {
entry.setValue(Boolean.TRUE);
}
// unselect only the given modules
for (IngestModuleAbstract unselectedModule : unselectedModules) {
getEntryForModule(unselectedModule).setValue(Boolean.FALSE);
}
// tell everyone about it
fireTableDataChanged();
}
public IngestModuleAbstract getModule(int row) {
return moduleData.get(row).getKey();
}
private Map.Entry<IngestModuleAbstract, Boolean> getEntryForModule(IngestModuleAbstract module) {
Map.Entry<IngestModuleAbstract, Boolean> entry = null;
for (Map.Entry<IngestModuleAbstract, Boolean> anEntry : moduleData) {
if (anEntry.getKey().equals(module)) {
entry = anEntry;
break;
}
}
return entry;
}
}
/**
@ -426,20 +365,15 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings;
List<String> tooltips = new ArrayList<>();
public ModulesTableRenderer() {
List<IngestModuleAbstract> modules = IngestManager.getDefault().enumerateAllModules();
for (IngestModuleAbstract ingestModuleAbstract : modules) {
tooltips.add(ingestModuleAbstract.getDescription());
for (IngestModuleTemplate moduleTemplate : moduleModels) {
tooltips.add(moduleTemplate.getModuleDescription());
}
}
@Override
public Component getTableCellRendererComponent(
JTable table, Object value,
boolean isSelected, boolean hasFocus,
int row, int column) {
public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) {
super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column);
if (column == 1) {
if (1 == column) {
setToolTipText(tooltips.get(row));
}
return this;

View File

@ -792,6 +792,10 @@ public class IngestManager {
modules.addAll(enumerateAbstractFileModules());
return modules;
}
List<IngestModuleFactory> getIngestModuleFactories() {
return moduleLoader.getIngestModuleFactories();
}
//data source worker to remove itself when complete or interrupted
void removeDataSourceIngestWorker(IngestDataSourceThread worker) {

View File

@ -91,7 +91,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
* code refactored
*/
final class IngestModuleLoader {
private ArrayList<IngestModuleFactory> moduleFactories = new ArrayList<>();
private static final String PIPELINE_CONFIG_XML = "pipeline_config.xml";
private static final String XSDFILE = "PipelineConfigSchema.xsd";
private String absFilePath;
@ -456,6 +456,10 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
return urls;
}
List<IngestModuleFactory> getIngestModuleFactories() {
return moduleFactories;
}
/**
* Auto-discover ingest modules in all platform modules that are "enabled"
* If discovered ingest module is not already in XML config, add it do
@ -466,6 +470,11 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
@SuppressWarnings("unchecked")
private void autodiscover() throws IngestModuleLoaderException {
Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class);
moduleFactories.addAll(factories);
// moduleFactories
// Use Lookup to find the other NBM modules. We'll later search them for ingest modules
Collection<? extends ModuleInfo> moduleInfos = Lookup.getDefault().lookupAll(ModuleInfo.class);
logger.log(Level.INFO, "Autodiscovery, found #platform modules: " + moduleInfos.size());

View File

@ -0,0 +1,61 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.io.Serializable;
/**
* RJCTODO
*/
public class IngestModuleTemplate {
private final IngestModuleFactory moduleFactory;
private Serializable ingestOptions = null;
private boolean enabled = true;
IngestModuleTemplate(IngestModuleFactory moduleFactory, Serializable ingestOptions, boolean enabled) {
this.moduleFactory = moduleFactory;
this.ingestOptions = ingestOptions;
this.enabled = enabled;
}
String getModuleDisplayName() {
return moduleFactory.getModuleDisplayName();
}
String getModuleDescription() {
return moduleFactory.getModuleDescription();
}
Serializable getIngestOptions() {
return ingestOptions;
}
void setIngestOptions(Serializable ingestOptions) {
this.ingestOptions = ingestOptions;
}
void setEnabled(boolean enabled) {
this.enabled = enabled;
}
boolean isEnabled() {
return enabled;
}
}