diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageAction.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageAction.java index a769194ed3..63221d3e8b 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageAction.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageAction.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,6 @@ package org.sleuthkit.autopsy.casemodule; -import org.sleuthkit.autopsy.ingest.IngestConfigurator; import java.awt.Component; import java.awt.Dialog; import java.awt.event.ActionEvent; @@ -36,12 +35,12 @@ import org.openide.DialogDisplayer; import org.openide.WizardDescriptor; import org.openide.util.ChangeSupport; import org.openide.util.HelpCtx; -import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; import org.openide.util.actions.Presenter; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.datamodel.Image; /** @@ -109,8 +108,7 @@ public final class AddImageAction extends CallableSystemAction implements Presen public void actionPerformed(ActionEvent e) { Logger.noteAction(AddImageAction.class); - final IngestConfigurator ingestConfig = Lookup.getDefault().lookup(IngestConfigurator.class); - if (null != ingestConfig && ingestConfig.isIngestRunning()) { + if (IngestManager.getDefault().isIngestRunning()) { final String msg = NbBundle.getMessage(this.getClass(), "AddImageAction.ingestConfig.ongoingIngest.msg"); if (JOptionPane.showConfirmDialog(null, msg, NbBundle.getMessage(this.getClass(), diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index 0e0726480b..f82a73b5c0 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -19,8 +19,8 @@ package org.sleuthkit.autopsy.casemodule; +import org.sleuthkit.autopsy.ingest.IngestJobLauncher; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.ingest.IngestConfigurator; import java.awt.Color; import java.awt.Component; import java.awt.Window; @@ -33,7 +33,6 @@ import javax.swing.SwingUtilities; import javax.swing.event.ChangeListener; import org.openide.WizardDescriptor; import org.openide.util.HelpCtx; -import org.openide.util.Lookup; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; @@ -47,7 +46,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel { private static final Logger logger = Logger.getLogger(AddImageWizardIngestConfigPanel.class.getName()); - private IngestConfigurator ingestConfig; + private IngestJobLauncher ingestConfig; /** * The visual component that displays this panel. If you need to access the * component from this class, just use getComponent(). @@ -74,8 +73,8 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel messages = ingestConfig.setContext(AddImageWizardIngestConfigPanel.class.getCanonicalName()); + ingestConfig = new IngestJobLauncher(AddImageWizardIngestConfigPanel.class.getCanonicalName()); + List messages = ingestConfig.getIngestJobConfigWarnings(); if (messages.isEmpty() == false) { StringBuilder warning = new StringBuilder(); for (String message : messages) { @@ -96,7 +95,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.PanelsingletonList(content)); - ingestDialog.display(); - } - }); + @Override + public void actionPerformed(ActionEvent e) { + final IngestDialog ingestDialog = new IngestDialog(); + ingestDialog.setDataSources(Collections.singletonList(content)); + ingestDialog.display(); + } + }); } //check if delete actions should be added @@ -141,7 +141,7 @@ class DirectoryTreeFilterNode extends FilterNode { } private static List getDetailActions(Content c) { - List actions = new ArrayList(); + List actions = new ArrayList<>(); actions.addAll(ExplorerNodeActionVisitor.getActions(c)); diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index c00b54811f..d5e82204b2 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -64,7 +64,7 @@ import org.sleuthkit.autopsy.datamodel.RootContentChildren; import org.sleuthkit.autopsy.datamodel.Views; import org.sleuthkit.autopsy.datamodel.ViewsNode; import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent; +import org.sleuthkit.autopsy.ingest.IngestManager.IngestEvent; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -587,7 +587,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // change in node selection else if (changed.equals(ExplorerManager.PROP_SELECTED_NODES)) { respondSelection((Node[]) oldValue, (Node[]) newValue); - } else if (changed.equals(IngestModuleEvent.DATA.toString())) { + } else if (changed.equals(IngestEvent.DATA.toString())) { final ModuleDataEvent event = (ModuleDataEvent) oldValue; if (event.getArtifactType() == BlackboardArtifact.ARTIFACT_TYPE.TSK_GEN_INFO) { return; @@ -598,7 +598,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat refreshTree(event.getArtifactType()); } }); - } else if (changed.equals(IngestModuleEvent.COMPLETED.toString())) { + } else if (changed.equals(IngestEvent.COMPLETED.toString())) { SwingUtilities.invokeLater(new Runnable() { @Override public void run() { @@ -606,7 +606,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat refreshTree(); } }); - } else if (changed.equals(IngestModuleEvent.CONTENT_CHANGED.toString())) { + } else if (changed.equals(IngestEvent.CONTENT_CHANGED.toString())) { SwingUtilities.invokeLater(new Runnable() { @Override public void run() { diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java index 810894b089..0ebce34e68 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java @@ -1,33 +1,32 @@ /* -* Sample module in the public domain. Feel free to use this as a template -* for your modules. -* -* Contact: Brian Carrier [carrier sleuthkit [dot] org] -* -* This is free and unencumbered software released into the public domain. -* -* Anyone is free to copy, modify, publish, use, compile, sell, or -* distribute this software, either in source code form or as a compiled -* binary, for any purpose, commercial or non-commercial, and by any -* means. -* -* In jurisdictions that recognize copyright laws, the author or authors -* of this software dedicate any and all copyright interest in the -* software to the public domain. We make this dedication for the benefit -* of the public at large and to the detriment of our heirs and -* successors. We intend this dedication to be an overt act of -* relinquishment in perpetuity of all present and future rights to this -* software under copyright law. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -* IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -* OTHER DEALINGS IN THE SOFTWARE. -*/ - + * Sample module in the public domain. Feel free to use this as a template + * for your modules. + * + * Contact: Brian Carrier [carrier sleuthkit [dot] org] + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ package org.sleuthkit.autopsy.examples; import java.util.List; @@ -35,10 +34,10 @@ import org.apache.log4j.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.casemodule.services.Services; -import org.sleuthkit.autopsy.ingest.IngestDataSourceWorkerController; -import org.sleuthkit.autopsy.ingest.IngestModuleDataSource; -import org.sleuthkit.autopsy.ingest.IngestModuleInit; -import org.sleuthkit.autopsy.ingest.PipelineContext; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModule; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleStatusHelper; +import org.sleuthkit.autopsy.ingest.IngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.FsContent; @@ -46,74 +45,40 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** - * Sample DataSource-level ingest module that doesn't do much at all. - * Just exists to show basic idea of these modules + * Sample data source ingest module that doesn't do much. Note that the + * IngestModuleAdapter abstract class could have been used as a base class to + * obtain default implementations of many of the DataSourceIngestModule methods. */ - public class SampleDataSourceIngestModule extends org.sleuthkit.autopsy.ingest.IngestModuleDataSource { - /* Data Source modules operate on a disk or set of logical files. They - * are passed in teh data source refernce and query it for things they want. - */ +// RJCTODO: Remove inheritance from IngestModuleAdapter to show full implementation of interface +// provide better documentation, and provide more extensive demonstration of how to +// use various ingest services. +class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSourceIngestModule { + + private static final Logger logger = Logger.getLogger(SampleDataSourceIngestModule.class); + @Override - public void process(PipelineContext pipelineContext, Content dataSource, IngestDataSourceWorkerController controller) { - + public ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) { Case case1 = Case.getCurrentCase(); SleuthkitCase sleuthkitCase = case1.getSleuthkitCase(); - + Services services = new Services(sleuthkitCase); - FileManager fm = services.getFileManager(); + FileManager fileManager = services.getFileManager(); try { - /* you can use the findFiles method in FileManager (or similar ones in - * SleuthkitCase to find files based only on their name. This - * one finds files that have a .doc extension. */ - List docFiles = fm.findFiles(dataSource, "%.doc"); + List docFiles = fileManager.findFiles(dataSource, "%.doc"); for (AbstractFile file : docFiles) { // do something with each doc file } - - /* We can also do more general queries with findFilesWhere, which - * allows us to make our own WHERE clause in the database. - */ - long currentTime = System.currentTimeMillis()/1000; - // go back 2 weeks - long minTime = currentTime - (14 * 24 * 60 * 60); + + long currentTime = System.currentTimeMillis() / 1000; + long minTime = currentTime - (14 * 24 * 60 * 60); // Go back two weeks. List otherFiles = sleuthkitCase.findFilesWhere("crtime > " + minTime); // do something with these files... - + } catch (TskCoreException ex) { - Logger log = Logger.getLogger(SampleDataSourceIngestModule.class); - log.fatal("Error retrieving files from database: " + ex.getLocalizedMessage()); - } - } - - @Override - public void init(IngestModuleInit initContext) throws IngestModuleException { - } - - @Override - public void complete() { - } - - @Override - public void stop() { - } - - @Override - public String getName() { - return "SampleDataSourceIngestModule"; - } - - @Override - public String getVersion() { - return "1.0"; - } - - @Override - public String getDescription() { - return "Doesn't do much"; - } - - @Override - public boolean hasBackgroundJobsRunning() { - return false; + logger.fatal("Error retrieving files from database: " + ex.getLocalizedMessage()); + return IngestModule.ProcessResult.OK; + } + + return IngestModule.ProcessResult.OK; } } diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java index a8ad161a3c..8510646957 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java @@ -32,9 +32,10 @@ package org.sleuthkit.autopsy.examples; import org.apache.log4j.Logger; import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; -import org.sleuthkit.autopsy.ingest.IngestModuleInit; -import org.sleuthkit.autopsy.ingest.PipelineContext; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -53,26 +54,15 @@ import org.sleuthkit.datamodel.TskData; * org.sleuthkit.autopsy.examples package. Either change the package or the * loading code to make this module actually run. */ -public class SampleFileIngestModule extends org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile { +// RJCTODO: Remove inheritance from IngestModuleAdapter to show full implementation of interface +// provide better documentation, and provide more extensive demonstration of how to +// use various ingest services. +class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestModule { private int attrId = -1; - private static SampleFileIngestModule defaultInstance = null; - - // Private to ensure Singleton status - private SampleFileIngestModule() { - } - - // File-level ingest modules are currently singleton -- this is required - public static synchronized SampleFileIngestModule getDefault() { - //defaultInstance is a private static class variable - if (defaultInstance == null) { - defaultInstance = new SampleFileIngestModule(); - } - return defaultInstance; - } @Override - public void init(IngestModuleInit initContext) throws IngestModuleException { + public void startUp(IngestJobContext initContext) { /* For this demo, we are going to make a private attribute to post our * results to the blackbaord with. There are many standard blackboard artifact * and attribute types and you should first consider using one of those before @@ -99,19 +89,18 @@ public class SampleFileIngestModule extends org.sleuthkit.autopsy.ingest.IngestM } @Override - public ProcessResult process(PipelineContext pipelineContext, AbstractFile abstractFile) { + public IngestModule.ProcessResult process(AbstractFile abstractFile) { // skip non-files if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { - return ProcessResult.OK; + return IngestModule.ProcessResult.OK; } // skip NSRL / known files if (abstractFile.getKnown() == TskData.FileKnown.KNOWN) { - return ProcessResult.OK; + return IngestModule.ProcessResult.OK; } - /* Do a non-sensical calculation of the number of 0x00 bytes * in the first 1024-bytes of the file. This is for demo * purposes only. @@ -128,7 +117,7 @@ public class SampleFileIngestModule extends org.sleuthkit.autopsy.ingest.IngestM if (attrId != -1) { // Make an attribute using the ID for the private type that we previously created. - BlackboardAttribute attr = new BlackboardAttribute(attrId, getName(), count); + BlackboardAttribute attr = new BlackboardAttribute(attrId, "SampleFileIngestModule", count); // RJCTODO: Set up factory with static module name function as example /* add it to the general info artifact. In real modules, you would likely have * more complex data types and be making more specific artifacts. @@ -137,38 +126,12 @@ public class SampleFileIngestModule extends org.sleuthkit.autopsy.ingest.IngestM art.addAttribute(attr); } - return ProcessResult.OK; + return IngestModule.ProcessResult.OK; } catch (TskCoreException ex) { Exceptions.printStackTrace(ex); - return ProcessResult.ERROR; + return IngestModule.ProcessResult.ERROR; } } - @Override - public void complete() { - } - - @Override - public void stop() { - } - - @Override - public String getVersion() { - return "1.0"; - } - - @Override - public String getName() { - return "SampleFileIngestModule"; - } - - @Override - public String getDescription() { - return "Doesn't do much"; - } - - @Override - public boolean hasBackgroundJobsRunning() { - return false; - } + // RJCTODO: Add a module factory with service provider annotation (commented out) } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index af1f7bea14..cc95e5b986 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -6,7 +6,6 @@ IngestMessageDetailsPanel.viewArtifactButton.text=Go to Result IngestMessageDetailsPanel.viewContentButton.text=Go to Directory IngestMessagePanel.sortByLabel.text=Sort by: IngestMessagePanel.sortByComboBox.toolTipText=Sort messages by time (chronological order) or message priority -IngestDialogPanel.advancedButton.text=Advanced IngestMessageDetailsPanel.messageDetailsPane.contentType=text/html IngestMessageDetailsPanel.messageDetailsPane.toolTipText= IngestMessagesToolbar.toolTipText= @@ -17,16 +16,14 @@ IngestMessagePanel.totalMessagesNameLabel.text=Total: IngestMessagePanel.totalMessagesNameVal.text=- IngestMessagePanel.totalUniqueMessagesNameLabel.text=Unique: IngestMessagePanel.totalUniqueMessagesNameVal.text=- -IngestDialogPanel.processUnallocCheckbox.text=Process Unallocated Space -IngestDialogPanel.processUnallocCheckbox.toolTipText=Processes unallocated space, such as deleted files. Produces more complete results, but it may take longer to process on large images. -DataSourceTask.toString.text=ScheduledTask'{'input\={0}, modules\={1}'}' -GeneralIngestConfigurator.modName.tbirdParser.text=Thunderbird Parser -GeneralIngestConfigurator.modName.mboxParser.text=MBox Parser -GeneralIngestConfigurator.modName.emailParser.text=Email Parser -GeneralIngestConfigurator.enabledMods.notFound.msg={0} was previously enabled, but could not be found -IngestDataSourceThread.displayName.text={0} dataSource id\:{1} -IngestDataSourceThread.progress.pending={0} (Pending...) -IngestDataSourceThread.progress.cancelling={0} (Cancelling...) +IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=Processes unallocated space, such as deleted files. Produces more complete results, but it may take longer to process on large images. +IngestJobConfigurationPanel.processUnallocCheckbox.text=Process Unallocated Space +IngestJobConfigurationPanel.advancedButton.text=Advanced +IngestJob.toString.text=ScheduledTask'{'input\={0}, modules\={1}'}' +IngestJobLauncher.modName.tbirdParser.text=Thunderbird Parser +IngestJobLauncher.modName.mboxParser.text=MBox Parser +IngestJobLauncher.modName.emailParser.text=Email Parser +IngestJobLauncher.enabledMods.notFound.msg={0} was previously enabled, but could not be found IngestDialog.title.text=Ingest Modules IngestDialog.startButton.title=Start IngestDialog.closeButton.title=Close @@ -49,12 +46,15 @@ IngestManager.toHtmlStr.totalErrs.text=Total errors\: {0} IngestManager.toHtmlStr.module.text=Module IngestManager.toHtmlStr.time.text=Time IngestManager.toHtmlStr.errors.text=Errors -IngestManager.IngestAbstractFileProcessor.displayName=File Ingest -IngestManager.IngestAbstractFileProcessor.process.cancelling={0} (Cancelling...) +IngestManager.FileTaskWorker.displayName=File Ingest +IngestManager.FileTaskWorker.process.cancelling={0} (Cancelling...) IngestManager.EnqueueWorker.displayName.text=Queueing Ingest IngestManager.EnqueueWorker.process.cancelling={0} (Cancelling...) +IngestManager.DataSourceTaskWorker.progress.pending={0} (Pending...) +IngestManager.DataSourceTaskWorker.progress.cancelling={0} (Cancelling...) IngestManager.datatSourceIngest.progress.text=DataSource Ingest {0} IngestManager.fileIngest.progress.text=File Ingest {0} +IngestJob.DataSourceIngestPipeline.displayName.text={0} processing {1} IngestMessage.toString.type.text=type\: {0} IngestMessage.toString.source.text=\ source\: {0} IngestMessage.toString.date.text=\ date\: {0} @@ -101,4 +101,3 @@ IngestScheduler.remove.exception.notSupported.msg=Not supported. IngestScheduler.DataSourceScheduler.exception.next.msg=There is no data source tasks in the queue, check hasNext() IngestScheduler.DataSourceScheduler.exception.remove.msg=Removing of scheduled data source ingest tasks is not supported. IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\: {0} -PipelineContext.toString.text=pipelineContext'{'task\={0}'}' diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java new file mode 100755 index 0000000000..5975990b5d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java @@ -0,0 +1,37 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.Content; + +/** + * Interface that must be implemented by all data source ingest modules. + */ +public interface DataSourceIngestModule extends IngestModule { + + /** + * Processes a data source. + * + * @param dataSource The data source to process. + * @param statusHelper A status helper to be used to report progress and + * detect ingest job cancellation. + * @return A result code indicating success or failure of the processing. + */ + ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper); +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java new file mode 100644 index 0000000000..db71c51fb7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java @@ -0,0 +1,87 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import javax.swing.SwingWorker; +import org.netbeans.api.progress.ProgressHandle; +import org.sleuthkit.datamodel.Content; + +/** + * Used by data source ingest modules to report progress and detect data source + * ingest job cancellation. + */ +public class DataSourceIngestModuleStatusHelper { + + private final SwingWorker worker; + private final ProgressHandle progress; + private final Content dataSource; + + DataSourceIngestModuleStatusHelper(SwingWorker worker, ProgressHandle progress, Content dataSource) { + this.worker = worker; + this.progress = progress; + this.dataSource = dataSource; + } + + /** + * Checks for ingest job cancellation. This should be polled by the module + * in its process() method. If the ingest task is canceled, the module + * should return from its process() method as quickly as possible. + * + * @return True if the task has been canceled, false otherwise. + */ + public boolean isCancelled() { + return worker.isCancelled(); + } + + /** + * Updates the progress bar and switches it to determinate mode. This should + * be called by the module as soon as the number of total work units + * required to process the data source is known. + * + * @param workUnits Total number of work units for the processing of the + * data source. + */ + public void switchToDeterminate(int workUnits) { + if (progress != null) { + progress.switchToDeterminate(workUnits); + } + } + + /** + * Switches the progress bar to indeterminate mode. This should be called if + * the total work units to process the data source is unknown. + */ + public void switchToIndeterminate() { + if (progress != null) { + progress.switchToIndeterminate(); + } + } + + /** + * Updates the progress bar with the number of work units performed, if in + * the determinate mode. + * + * @param workUnits Number of work units performed so far by the module. + */ + public void progress(int workUnits) { + if (progress != null) { + progress.progress(dataSource.getName(), workUnits); + } + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceTask.java deleted file mode 100644 index d35f4a89b7..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceTask.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2013-2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.sleuthkit.autopsy.ingest; - -import java.util.List; -import org.openide.util.NbBundle; -import org.sleuthkit.datamodel.Content; - -/** - * Represents a data source-level task to schedule and analyze. - * Children of the data will also be scheduled. - * - * @param T type of Ingest Module / Pipeline (file or data source content) associated with this task - */ -class DataSourceTask { - private Content input; - private List modules; - private boolean processUnallocated; - private PipelineContext pipelineContext; - - public DataSourceTask(Content input, List modules, boolean processUnallocated) { - this.input = input; - this.modules = modules; - this.processUnallocated = processUnallocated; - pipelineContext = new PipelineContext(this); - } - - public Content getContent() { - return input; - } - - public PipelineContext getPipelineContext() { - return pipelineContext; - } - - public List getModules() { - return modules; - } - - /** - * Returns value of if unallocated space should be analyzed (and scheduled) - * @return True if pipeline should process unallocated space. - */ - boolean isProcessUnalloc() { - return processUnallocated; - } - - // @@@ BC: I think this should go away. - void addModules(List newModules) { - for (T newModule : newModules) { - if (!modules.contains(newModule)) { - modules.add(newModule); - } - } - } - - @Override - public String toString() { - return NbBundle.getMessage(this.getClass(), "DataSourceTask.toString.text", input, modules); - } - - /** - * Two scheduled tasks are equal when the content and modules are the same. - * This enables us not to enqueue the equal schedules tasks twice into the - * queue/set - * - * @param obj - * @return - */ - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - @SuppressWarnings("unchecked") - final DataSourceTask other = (DataSourceTask) obj; - if (this.input != other.input && (this.input == null || !this.input.equals(other.input))) { - return false; - } - if (this.modules != other.modules && (this.modules == null || !this.modules.equals(other.modules))) { - return false; - } - - return true; - } -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleInit.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java old mode 100644 new mode 100755 similarity index 51% rename from Core/src/org/sleuthkit/autopsy/ingest/IngestModuleInit.java rename to Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java index 013a123065..b212820424 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleInit.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2012 Basis Technology Corp. + * + * Copyright 2014 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,32 +18,18 @@ */ package org.sleuthkit.autopsy.ingest; +import org.sleuthkit.datamodel.AbstractFile; + /** - * - * Context passed to a module at initialization time. - * It may contain module configuration required to initialize some modules. + * Interface that must be implemented by all file ingest modules. */ -public class IngestModuleInit { - -// private String moduleArgs; +public interface FileIngestModule extends IngestModule { /** - * Get module arguments - * @return module args string, used by some modules + * Processes a file. + * + * @param file The file. + * @return A result code indicating success or failure of the processing. */ -// public String getModuleArgs() { -// return moduleArgs; -// } - - /** - * Sets module args. string (only used by module pipeline) - * @param moduleArgs arguments to set for the module - */ -// void setModuleArgs(String moduleArgs) { -// this.moduleArgs = moduleArgs; -// } -// - - - + ProcessResult process(AbstractFile file); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java deleted file mode 100644 index b96ad9cd49..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2013 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import javax.swing.JPanel; - -import org.openide.util.NbBundle; -import org.openide.util.lookup.ServiceProvider; -import org.sleuthkit.autopsy.coreutils.ModuleSettings; -import org.sleuthkit.datamodel.Content; - -@ServiceProvider(service = IngestConfigurator.class) -public class GeneralIngestConfigurator implements IngestConfigurator { - - public static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; - public static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; - public static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; - private List contentToIngest; - private IngestManager manager; - private IngestDialogPanel ingestDialogPanel; - private String moduleContext; - - public GeneralIngestConfigurator() { - this.moduleContext = IngestManager.MODULE_PROPERTIES; - ingestDialogPanel = new IngestDialogPanel(); - ingestDialogPanel.setContext(moduleContext); - manager = IngestManager.getDefault(); - } - - @Override - public List setContext(String context) { - moduleContext = context; - ingestDialogPanel.setContext(moduleContext); - return loadSettingsForContext(); - } - - private List loadSettingsForContext() { - List messages = new ArrayList<>(); - List allModules = IngestManager.getDefault().enumerateAllModules(); - - // If there is no enabled ingest modules setting for this user, default to enabling all - // of the ingest modules the IngestManager has loaded. - if (ModuleSettings.settingExists(moduleContext, ENABLED_INGEST_MODULES_KEY) == false) { - String defaultSetting = moduleListToCsv(allModules); - ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, defaultSetting); - } - - String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", "); - ArrayList enabledList = new ArrayList<>(Arrays.asList(enabledModuleNames)); - - // Check for modules that are missing from the config file - - String[] disabledModuleNames = null; - // Older config files won't have the disabled list, so don't assume it exists - if (ModuleSettings.settingExists(moduleContext, DISABLED_INGEST_MODULES_KEY)) { - disabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY).split(", "); - } - - for (IngestModuleAbstract module : allModules) { - boolean found = false; - - // Check enabled first - for (String moduleName : enabledModuleNames) { - if (module.getName().equals(moduleName)) { - found = true; - break; - } - } - - // Then check disabled - if (!found && (disabledModuleNames != null)) { - for (String moduleName : disabledModuleNames) { - if (module.getName().equals(moduleName)) { - found = true; - break; - } - } - } - - if (!found) { - enabledList.add(module.getName()); - // It will get saved to file later - } - } - - // Get the enabled ingest modules setting, check for missing modules, and pass the setting to - // the UI component. - List enabledModules = new ArrayList<>(); - for (String moduleName : enabledList) { - if (moduleName.equals( - NbBundle.getMessage(this.getClass(), "GeneralIngestConfigurator.modName.tbirdParser.text")) - || moduleName.equals( - NbBundle.getMessage(this.getClass(), "GeneralIngestConfigurator.modName.mboxParser.text"))) { - moduleName = NbBundle.getMessage(this.getClass(), "GeneralIngestConfigurator.modName.emailParser.text"); - } - - IngestModuleAbstract moduleFound = null; - for (IngestModuleAbstract module : allModules) { - if (moduleName.equals(module.getName())) { - moduleFound = module; - break; - } - } - if (moduleFound != null) { - enabledModules.add(moduleFound); - } - else { - messages.add(NbBundle.getMessage(this.getClass(), "GeneralIngestConfigurator.enabledMods.notFound.msg", - moduleName)); - } - } - ingestDialogPanel.setEnabledIngestModules(enabledModules); - - // If there is no process unallocated space flag setting, default it to false. - if (ModuleSettings.settingExists(moduleContext, PARSE_UNALLOC_SPACE_KEY) == false) { - ModuleSettings.setConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY, "false"); - } - - // Get the process unallocated space flag setting and pass it to the UI component. - boolean processUnalloc = Boolean.parseBoolean(ModuleSettings.getConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY)); - ingestDialogPanel.setProcessUnallocSpaceEnabled(processUnalloc); - - return messages; - } - - @Override - public JPanel getIngestConfigPanel() { - // Note that this panel allows for selecting modules for the ingest process, - // specifying the process unallocated space flag, and also specifying settings - // for a selected ingest module. - return ingestDialogPanel; - } - - @Override - public void save() { - // Save the user's configuration of the set of enabled ingest modules. - String enabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getModulesToStart()); - ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, enabledModulesCsvList); - - // Save the user's configuration of the set of disabled ingest modules. - String disabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getDisabledModules()); - ModuleSettings.setConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY, disabledModulesCsvList); - - // Save the user's setting for the process unallocated space flag. - String processUnalloc = Boolean.toString(ingestDialogPanel.processUnallocSpaceEnabled()); - ModuleSettings.setConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY, processUnalloc); - - // Save the user's configuration of the currently selected ingest module. - IngestModuleAbstract currentModule = ingestDialogPanel.getCurrentIngestModule(); - if (currentModule != null && currentModule.hasSimpleConfiguration()) { - currentModule.saveSimpleConfiguration(); - } - } - - private static String moduleListToCsv(List lst) { - if (lst == null || lst.isEmpty()) { - return ""; - } - - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < lst.size() - 1; ++i) { - sb.append(lst.get(i).getName()).append(", "); - } - - // and the last one - sb.append(lst.get(lst.size() - 1).getName()); - - return sb.toString(); - } - - @Override - public void setContent(List inputContent) { - this.contentToIngest = inputContent; - } - - @Override - public void start() { - // Get the list of ingest modules selected by the user. - List modulesToStart = ingestDialogPanel.getModulesToStart(); - - // Get the user's selection of whether or not to process unallocated space. - manager.setProcessUnallocSpace(ingestDialogPanel.processUnallocSpaceEnabled()); - - if (!modulesToStart.isEmpty() && contentToIngest != null) { - // Queue the ingest process. - manager.scheduleDataSource(modulesToStart, contentToIngest); - } - } - - @Override - public boolean isIngestRunning() { - return manager.isIngestRunning(); - } -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GetFilesContentVisitor.java b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesContentVisitor.java index 65290a5b3c..2a9ac8ced4 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/GetFilesContentVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesContentVisitor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,21 +25,17 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentVisitor; import org.sleuthkit.datamodel.Directory; -import org.sleuthkit.datamodel.File; -import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.VirtualDirectory; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.datamodel.Volume; import org.sleuthkit.datamodel.VolumeSystem; -import org.sleuthkit.datamodel.LayoutFile; /** - * Abstract visitor for getting all the files from content - * TODO should be moved to utility module (needs resolve cyclic deps) + * Abstract visitor for getting all the files from content. */ - abstract class GetFilesContentVisitor implements ContentVisitor> { +abstract class GetFilesContentVisitor implements ContentVisitor> { private static final Logger logger = Logger.getLogger(GetFilesContentVisitor.class.getName()); @@ -47,7 +43,7 @@ import org.sleuthkit.datamodel.LayoutFile; public Collection visit(VirtualDirectory ld) { return getAllFromChildren(ld); } - + @Override public Collection visit(Directory drctr) { return getAllFromChildren(drctr); @@ -69,13 +65,14 @@ import org.sleuthkit.datamodel.LayoutFile; } /** - * Aggregate all the matches from visiting the children Content objects of the - * one passed - * @param parent - * @return + * Aggregate all the matches from visiting the children Content objects of + * a parent Content object. + * + * @param parent A content object. + * @return The child files of the content. */ protected Collection getAllFromChildren(Content parent) { - Collection all = new ArrayList(); + Collection all = new ArrayList<>(); try { for (Content child : parent.getChildren()) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestConfigurator.java deleted file mode 100644 index 94ef6924b9..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestConfigurator.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011-2013 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import java.util.List; -import javax.swing.JPanel; -import org.sleuthkit.datamodel.Content; - -/** - * Instances of this class provide the following services: - * 1. A way to save and load the ingest process configuration settings for a - * given ingest process context. - * 2. A UI component for configuring ingest process settings. - * 3. A way to specify input content and start the ingest process for a - * given ingest process context. - */ -// @@@ This interface needs to be re-designed. An interface for allowing the -// authors of ingest modules to expose context sensitive module configuration -// settings needs to be provided; there also needs to be a way for users to -// configure the ingest process that uses those modules. These are separate -// concerns; likewise, kicking off an ingest process for particular content in -// a particular context is a separate concern. -public interface IngestConfigurator { - /** - * Specifies the ingest process context for the purpose of choosing, saving, - * and loading ingest process configuration settings; also determines what - * configuration settings will be in effect if the setContent() and start() - * methods are called to start the ingest process for some content specified - * using the setContent() method. - * @return A list, possibly empty, of messages describing errors that - * occurred when loading the configuration settings. - */ - public List setContext(String contextName); - - /** - * Provides a UI component for choosing ingest process configuration - * settings for the ingest process context specified using the setContext() - * method. - */ - JPanel getIngestConfigPanel(); - - /** - * Saves the ingest process configuration settings for the ingest process - * context specified using the setContext() method. - */ - void save(); - - /** - * Sets the input content for an ingest process prior to calling start() to - * run the process using the process configuration settings for the context - * specified using setContext(). - */ - void setContent(List inputContent); - - /** - * Starts (queues) the ingest process for the content specified using the - * setContent() method, using the configuration settings corresponding to - * the ingest process context specified using the setContext() method. - */ - void start(); - - /** - * Returns true if any ingest process is running, false otherwise. - * Note that the running process may or may not be the process started - * (queued) by an invocation of the start() method. - */ - boolean isIngestRunning(); -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDataSourceThread.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestDataSourceThread.java deleted file mode 100644 index 217a568344..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDataSourceThread.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -//ingester worker for DataSource queue -import java.awt.EventQueue; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.logging.Level; - -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import javax.swing.SwingWorker; -import org.netbeans.api.progress.ProgressHandle; -import org.netbeans.api.progress.ProgressHandleFactory; -import org.openide.util.Cancellable; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.coreutils.StopWatch; -import org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstract.IngestModuleException; -import org.sleuthkit.datamodel.Content; - -/** - * Worker thread that runs a data source-level ingest module (image, file set virt dir, etc). - * Used to process only a single data-source and single module. - */ - class IngestDataSourceThread extends SwingWorker { - - private final Logger logger = Logger.getLogger(IngestDataSourceThread.class.getName()); - private ProgressHandle progress; - private final PipelineContextpipelineContext; - private final Content dataSource; - private final IngestModuleDataSource module; - private IngestDataSourceWorkerController controller; - private final IngestManager manager; - private final IngestModuleInit init; - private boolean inited; - //current method of enqueuing data source ingest modules with locks and internal lock queue - //ensures that we init, run and complete a single data source ingest module at a time - //uses fairness policy to run them in order enqueued - //TODO use a real queue and manager to allow multiple different modules to run in parallel - private static final Lock dataSourceIngestModuleLock = new ReentrantReadWriteLock(true).writeLock(); - - IngestDataSourceThread(IngestManager manager, PipelineContextpipelineContext, Content dataSource, IngestModuleDataSource module, IngestModuleInit init) { - this.manager = manager; - this.pipelineContext = pipelineContext; - this.dataSource = dataSource; - this.module = module; - this.init = init; - this.inited = false; - } - - PipelineContextgetContext() { - return pipelineContext; - } - - Content getContent() { - return pipelineContext.getDataSourceTask().getContent(); - } - - IngestModuleDataSource getModule() { - return module; - } - - public void init() throws IngestModuleException{ - - logger.log(Level.INFO, "Initializing module: " + module.getName()); - try { - module.init(init); - inited = true; - } catch (IngestModuleException e) { - logger.log(Level.INFO, "Failed initializing module: " + module.getName() + ", will not run."); - //will not run - inited = false; - throw e; - } - } - - @Override - protected Void doInBackground() throws Exception { - - logger.log(Level.INFO, "Pending module: " + module.getName()); - - final String displayName = NbBundle.getMessage(this.getClass(), "IngestDataSourceThread.displayName.text", - module.getName(), - dataSource.getId()); - progress = ProgressHandleFactory.createHandle( - NbBundle.getMessage(this.getClass(), "IngestDataSourceThread.progress.pending", displayName), new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "DataSource ingest module " + module.getName() + " cancelled by user."); - if (progress != null) { - progress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestDataSourceThread.progress.cancelling", displayName)); - } - return IngestDataSourceThread.this.cancel(true); - } - }); - progress.start(); - progress.switchToIndeterminate(); - - dataSourceIngestModuleLock.lock(); - try { - if (this.isCancelled()) { - logger.log(Level.INFO, "Cancelled while pending, module: " + module.getName()); - return Void.TYPE.newInstance(); - } - logger.log(Level.INFO, "Starting module: " + module.getName()); - logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); - progress.setDisplayName(displayName); - - if (inited == false) { - logger.log(Level.INFO, "Module wasn't initialized, will not run: " + module.getName()); - return Void.TYPE.newInstance(); - } - logger.log(Level.INFO, "Starting processing of module: " + module.getName()); - - controller = new IngestDataSourceWorkerController(this, progress); - - if (isCancelled()) { - logger.log(Level.INFO, "Terminating DataSource ingest module " + module.getName() + " due to cancellation."); - return Void.TYPE.newInstance(); - } - final StopWatch timer = new StopWatch(); - timer.start(); - try { - module.process(pipelineContext, dataSource, controller); - } catch (Exception e) { - logger.log(Level.WARNING, "Exception in module: " + module.getName() + " DataSource: " + dataSource.getName(), e); - } finally { - timer.stop(); - logger.log(Level.INFO, "Done processing of module: " + module.getName() - + " took " + timer.getElapsedTimeSecs() + " secs. to process()"); - - - //cleanup queues (worker and DataSource/module) - manager.removeDataSourceIngestWorker(this); - - if (!this.isCancelled()) { - logger.log(Level.INFO, "Module " + module.getName() + " completed"); - try { - module.complete(); - } catch (Exception e) { - logger.log(Level.INFO, "Error completing the module " + module.getName(), e); - } - IngestManager.fireModuleEvent(IngestModuleEvent.COMPLETED.toString(), module.getName()); - } else { - logger.log(Level.INFO, "Module " + module.getName() + " stopped"); - try { - module.stop(); - } catch (Exception e) { - logger.log(Level.INFO, "Error stopping the module" + module.getName(), e); - } - IngestManager.fireModuleEvent(IngestModuleEvent.STOPPED.toString(), module.getName()); - } - - } - return Void.TYPE.newInstance(); - } finally { - //release the lock so next module can run - dataSourceIngestModuleLock.unlock(); - EventQueue.invokeLater(new Runnable() { - @Override - public void run() { - progress.finish(); - } - }); - logger.log(Level.INFO, "Done running module: " + module.getName()); - logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDataSourceWorkerController.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestDataSourceWorkerController.java deleted file mode 100644 index 052562e0f5..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDataSourceWorkerController.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import org.netbeans.api.progress.ProgressHandle; - -/** - * Controller for DataSource level ingest modules - * Used by modules to check task status and to post progress to - */ -public class IngestDataSourceWorkerController { - - private IngestDataSourceThread worker; - private ProgressHandle progress; - - /** - * Instantiate the controller for the worker - * @param worker underlying DataSource ingest thread - * @param progress the progress handle - */ - IngestDataSourceWorkerController(IngestDataSourceThread worker, ProgressHandle progress) { - this.worker = worker; - this.progress = progress; - } - - /** - * Check if the task has been cancelled. This should be polled by the module periodically - * And the module needs to act, i.e. break out of its processing loop and call its stop() to cleanup - * - * @return true if the task has been cancelled, false otherwise - */ - public boolean isCancelled() { - return worker.isCancelled(); - } - - /** - * Update the progress bar and switch to determinate mode once number of total work units is known - * @param workUnits total number of work units for the DataSource ingest task - */ - public void switchToDeterminate(int workUnits) { - if (progress != null) { - progress.switchToDeterminate(workUnits); - } - } - - /** - * Update the progress bar and switch to non determinate mode if number of work units is not known - */ - public void switchToInDeterminate() { - if (progress != null) { - progress.switchToIndeterminate(); - } - } - - /** - * Update the progress bar with the number of work units performed, if in the determinate mode - * @param workUnits number of work units performed so far by the module - */ - public void progress(int workUnits) { - if (progress != null) { - progress.progress(worker.getContent().getName(), workUnits); - } - } -} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialog.java index b6cedc099f..4c5aa4b28b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialog.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2013 Basis Technology Corp. + * Copyright 2013-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,7 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; +import java.util.ArrayList; import java.util.List; import javax.swing.BoxLayout; import javax.swing.JButton; @@ -32,24 +33,24 @@ import javax.swing.JDialog; import javax.swing.JFrame; import javax.swing.JOptionPane; import javax.swing.JPanel; - import org.openide.util.NbBundle; import org.sleuthkit.datamodel.Content; /** - * Dialog box that allows ingest modules to be run on an image. - * Used outside of the wizards. + * Dialog box that allows ingest modules to be run on a data source. Used + * outside of the wizards. */ -public class IngestDialog extends JDialog { - +public final class IngestDialog extends JDialog { + private static final String TITLE = NbBundle.getMessage(IngestDialog.class, "IngestDialog.title.text"); private static Dimension DIMENSIONS = new Dimension(500, 300); - private IngestConfigurator ingestConfigurator; - + private List dataSources = new ArrayList<>(); + private IngestJobLauncher ingestJobLauncher; + public IngestDialog(JFrame frame, String title, boolean modal) { super(frame, title, modal); - ingestConfigurator = new GeneralIngestConfigurator(); - List messages = ingestConfigurator.setContext(IngestDialog.class.getCanonicalName()); + ingestJobLauncher = new IngestJobLauncher(IngestDialog.class.getCanonicalName()); + List messages = ingestJobLauncher.getIngestJobConfigWarnings(); if (messages.isEmpty() == false) { StringBuilder warning = new StringBuilder(); for (String message : messages) { @@ -58,8 +59,8 @@ public class IngestDialog extends JDialog { JOptionPane.showMessageDialog(null, warning.toString()); } } - - public IngestDialog(){ + + public IngestDialog() { this(new JFrame(TITLE), TITLE, true); } @@ -78,51 +79,49 @@ public class IngestDialog extends JDialog { // set the location of the popUp Window on the center of the screen setLocation((screenDimension.width - w) / 2, (screenDimension.height - h) / 2); - add(ingestConfigurator.getIngestConfigPanel(), BorderLayout.PAGE_START); + add(ingestJobLauncher.getIngestJobConfigPanel(), BorderLayout.PAGE_START); JButton startButton = new JButton(NbBundle.getMessage(this.getClass(), "IngestDialog.startButton.title")); JButton closeButton = new JButton(NbBundle.getMessage(this.getClass(), "IngestDialog.closeButton.title")); startButton.addActionListener(new ActionListener() { - @Override public void actionPerformed(ActionEvent e) { - ingestConfigurator.save(); - ingestConfigurator.start(); + ingestJobLauncher.saveIngestJobConfig(); + ingestJobLauncher.startIngestJobs(dataSources); close(); } }); closeButton.addActionListener(new ActionListener() { - @Override public void actionPerformed(ActionEvent e) { - ingestConfigurator.save(); + ingestJobLauncher.saveIngestJobConfig(); close(); } }); this.addWindowListener(new WindowAdapter() { - @Override public void windowClosing(WindowEvent e) { - ingestConfigurator.save(); + ingestJobLauncher.saveIngestJobConfig(); close(); } }); JPanel buttonPanel = new JPanel(); buttonPanel.setLayout(new BoxLayout(buttonPanel, BoxLayout.LINE_AXIS)); - buttonPanel.add(new javax.swing.Box.Filler(new Dimension(10,10), new Dimension(10,10), new Dimension(10,10))); + buttonPanel.add(new javax.swing.Box.Filler(new Dimension(10, 10), new Dimension(10, 10), new Dimension(10, 10))); buttonPanel.add(startButton); - buttonPanel.add(new javax.swing.Box.Filler(new Dimension(10,10), new Dimension(10,10), new Dimension(10,10))); + buttonPanel.add(new javax.swing.Box.Filler(new Dimension(10, 10), new Dimension(10, 10), new Dimension(10, 10))); buttonPanel.add(closeButton); add(buttonPanel, BorderLayout.LINE_START); - + pack(); setResizable(false); setVisible(true); } - - public void setContent(List inputContent) { - ingestConfigurator.setContent(inputContent); - } - + + public void setDataSources(List inputContent) { + dataSources.clear(); + dataSources.addAll(inputContent); + } + /** * Closes the Ingest dialog */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java new file mode 100644 index 0000000000..90507048ee --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -0,0 +1,389 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import javax.swing.SwingWorker; +import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; + +/** + * Encapsulates a data source and the ingest module pipelines to be used to + * ingest the data source. + */ +final class IngestJob { + + private final long id; + private final Content dataSource; + private final List ingestModuleTemplates; + private final boolean processUnallocatedSpace; + private final HashMap fileIngestPipelines = new HashMap<>(); + private final HashMap dataSourceIngestPipelines = new HashMap<>(); + private FileIngestPipeline initialFileIngestPipeline = null; + private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; + private boolean cancelled; + + IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { + this.id = id; + this.dataSource = dataSource; + this.ingestModuleTemplates = ingestModuleTemplates; + this.processUnallocatedSpace = processUnallocatedSpace; + this.cancelled = false; + } + + long getId() { + return id; + } + + Content getDataSource() { + return dataSource; + } + + boolean shouldProcessUnallocatedSpace() { + return processUnallocatedSpace; + } + + synchronized void cancel() { + cancelled = true; + } + + synchronized boolean isCancelled() { + return cancelled; + } + + synchronized List startUpIngestPipelines() { + // Create a per thread instance of each pipeline type right now to make + // (reasonably) sure that the ingest modules can be started. + initialDataSourceIngestPipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); + initialFileIngestPipeline = new FileIngestPipeline(this, ingestModuleTemplates); + List errors = new ArrayList<>(); + errors.addAll(initialDataSourceIngestPipeline.startUp()); + errors.addAll(initialFileIngestPipeline.startUp()); + return errors; + } + + synchronized DataSourceIngestPipeline getDataSourceIngestPipelineForThread(long threadId) { + DataSourceIngestPipeline pipeline; + if (initialDataSourceIngestPipeline != null) { + pipeline = initialDataSourceIngestPipeline; + initialDataSourceIngestPipeline = null; + dataSourceIngestPipelines.put(threadId, pipeline); + } else if (!dataSourceIngestPipelines.containsKey(threadId)) { + pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); + pipeline.startUp(); + dataSourceIngestPipelines.put(threadId, pipeline); + } else { + pipeline = dataSourceIngestPipelines.get(threadId); + } + return pipeline; + } + + synchronized FileIngestPipeline getFileIngestPipelineForThread(long threadId) { + FileIngestPipeline pipeline; + if (initialFileIngestPipeline != null) { + pipeline = initialFileIngestPipeline; + initialFileIngestPipeline = null; + fileIngestPipelines.put(threadId, pipeline); + } else if (!fileIngestPipelines.containsKey(threadId)) { + pipeline = new FileIngestPipeline(this, ingestModuleTemplates); + pipeline.startUp(); + fileIngestPipelines.put(threadId, pipeline); + } else { + pipeline = fileIngestPipelines.get(threadId); + } + return pipeline; + } + + synchronized List releaseIngestPipelinesForThread(long threadId) { + List errors = new ArrayList<>(); + + DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId); + if (dataSourceIngestPipeline != null) { + errors.addAll(dataSourceIngestPipeline.shutDown(cancelled)); + } + this.dataSourceIngestPipelines.remove(threadId); + + FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId); + if (fileIngestPipeline != null) { + errors.addAll(fileIngestPipeline.shutDown(cancelled)); + } + this.fileIngestPipelines.remove(threadId); + + return errors; + } + + synchronized boolean areIngestPipelinesShutDown() { + return (dataSourceIngestPipelines.isEmpty() && fileIngestPipelines.isEmpty()); + } + + /** + * A data source ingest pipeline composed of a sequence of data source ingest + * modules constructed from ingest module templates. + */ + static final class DataSourceIngestPipeline { + + private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName()); + private final IngestJob task; + private final List moduleTemplates; + private List modules = new ArrayList<>(); + + private DataSourceIngestPipeline(IngestJob task, List moduleTemplates) { + this.task = task; + this.moduleTemplates = moduleTemplates; + } + + private List startUp() { + List errors = new ArrayList<>(); + + // Create an ingest module instance from each ingest module template + // that has an ingest module factory capable of making data source + // ingest modules. Map the module class names to the module instance + // to allow the modules to be put in the sequence indicated by the + // ingest pipelines configuration. + Map modulesByClass = new HashMap<>(); + for (IngestModuleTemplate template : moduleTemplates) { + if (template.isDataSourceIngestModuleTemplate()) { + DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); + IngestJobContext context = new IngestJobContext(task); + try { + module.startUp(context); + modulesByClass.put(module.getClassName(), module); + IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), module.getDisplayName()); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + } + } + + // Establish the module sequence of the core ingest modules + // indicated by the ingest pipeline configuration, adding any + // additional modules found in the global lookup to the end of the + // pipeline in arbitrary order. + List pipelineConfig = IngestPipelinesConfiguration.getInstance().getDataSourceIngestPipelineConfig(); + for (String moduleClassName : pipelineConfig) { + if (modulesByClass.containsKey(moduleClassName)) { + modules.add(modulesByClass.remove(moduleClassName)); + } + } + for (DataSourceIngestModuleDecorator module : modulesByClass.values()) { + modules.add(module); + } + + return errors; + } + + List process(SwingWorker worker, ProgressHandle progress) { + List errors = new ArrayList<>(); + Content dataSource = this.task.getDataSource(); + logger.log(Level.INFO, "Processing data source {0}", dataSource.getName()); + for (DataSourceIngestModuleDecorator module : this.modules) { + try { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.DataSourceIngestPipeline.displayName.text", module.getDisplayName(), dataSource.getName()); + progress.setDisplayName(displayName); + module.process(dataSource, new DataSourceIngestModuleStatusHelper(worker, progress, dataSource)); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (task.isCancelled()) { + break; + } + } + return errors; + } + + private List shutDown(boolean ingestJobCancelled) { + List errors = new ArrayList<>(); + for (DataSourceIngestModuleDecorator module : this.modules) { + try { + module.shutDown(ingestJobCancelled); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } finally { + IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); + } + } + return errors; + } + + private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + + private final DataSourceIngestModule module; + private final String displayName; + + DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { + this.module = module; + this.displayName = displayName; + } + + String getClassName() { + return module.getClass().getCanonicalName(); + } + + String getDisplayName() { + return displayName; + } + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + module.startUp(context); + } + + @Override + public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) { + return module.process(dataSource, statusHelper); + } + + @Override + public void shutDown(boolean ingestJobWasCancelled) { + module.shutDown(ingestJobWasCancelled); + } + } + } + + /** + * A file ingest pipeline composed of a sequence of file ingest modules + * constructed from ingest module templates. + */ + static final class FileIngestPipeline { + + private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName()); + private final IngestJob task; + private final List moduleTemplates; + private List modules = new ArrayList<>(); + + private FileIngestPipeline(IngestJob task, List moduleTemplates) { + this.task = task; + this.moduleTemplates = moduleTemplates; + } + + private List startUp() { + List errors = new ArrayList<>(); + + // Create an ingest module instance from each ingest module template + // that has an ingest module factory capable of making data source + // ingest modules. Map the module class names to the module instance + // to allow the modules to be put in the sequence indicated by the + // ingest pipelines configuration. + Map modulesByClass = new HashMap<>(); + for (IngestModuleTemplate template : moduleTemplates) { + if (template.isFileIngestModuleTemplate()) { + FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); + IngestJobContext context = new IngestJobContext(task); + try { + module.startUp(context); + modulesByClass.put(module.getClassName(), module); + IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), template.getModuleName()); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + } + } + + // Establish the module sequence of the core ingest modules + // indicated by the ingest pipeline configuration, adding any + // additional modules found in the global lookup to the end of the + // pipeline in arbitrary order. + List pipelineConfig = IngestPipelinesConfiguration.getInstance().getFileIngestPipelineConfig(); + for (String moduleClassName : pipelineConfig) { + if (modulesByClass.containsKey(moduleClassName)) { + modules.add(modulesByClass.remove(moduleClassName)); + } + } + for (FileIngestModuleDecorator module : modulesByClass.values()) { + modules.add(module); + } + + return errors; + } + + List process(AbstractFile file) { + List errors = new ArrayList<>(); + Content dataSource = this.task.getDataSource(); + logger.log(Level.INFO, String.format("Processing {0} from {1}", file.getName(), dataSource.getName())); + for (FileIngestModuleDecorator module : this.modules) { + try { + module.process(file); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (task.isCancelled()) { + break; + } + } + file.close(); + IngestManager.fireFileDone(file.getId()); + return errors; + } + + private List shutDown(boolean ingestJobCancelled) { + List errors = new ArrayList<>(); + for (FileIngestModuleDecorator module : this.modules) { + try { + module.shutDown(ingestJobCancelled); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } finally { + IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); + } + } + return errors; + } + + private static class FileIngestModuleDecorator implements FileIngestModule { + + private final FileIngestModule module; + private final String displayName; + + FileIngestModuleDecorator(FileIngestModule module, String displayName) { + this.module = module; + this.displayName = displayName; + } + + String getClassName() { + return module.getClass().getCanonicalName(); + } + + String getDisplayName() { + return displayName; + } + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + module.startUp(context); + } + + @Override + public IngestModule.ProcessResult process(AbstractFile file) { + return module.process(file); + } + + @Override + public void shutDown(boolean ingestJobWasCancelled) { + module.shutDown(ingestJobWasCancelled); + } + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.form b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form similarity index 95% rename from Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.form rename to Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form index ad7862b24a..ae6e4c7e93 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.form +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form @@ -134,7 +134,7 @@ - + @@ -201,10 +201,10 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java similarity index 64% rename from Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java rename to Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java index ff6ef874e9..20b9f50e3d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,10 +23,8 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; -import java.util.Map; import javax.swing.JTable; import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; @@ -35,62 +33,56 @@ import javax.swing.table.AbstractTableModel; import javax.swing.table.DefaultTableCellRenderer; import javax.swing.table.TableColumn; import org.sleuthkit.autopsy.corecomponents.AdvancedConfigurationDialog; -import org.sleuthkit.autopsy.coreutils.ModuleSettings; /** - * main configuration panel for all ingest modules, reusable JPanel component + * User interface component to allow a user to set ingest module options and + * enable/disable the modules. */ - class IngestDialogPanel extends javax.swing.JPanel { +class IngestJobConfigurationPanel extends javax.swing.JPanel { - private IngestModuleAbstract currentModule; - private ModulesTableModel tableModel; - private String context; + private List modules = new ArrayList<>(); + private boolean processUnallocatedSpace = false; + private IngestModuleModel selectedModule = null; - /** - * Creates new form IngestDialogPanel - */ - public IngestDialogPanel() { - tableModel = new ModulesTableModel(); - context = ModuleSettings.DEFAULT_CONTEXT; + IngestJobConfigurationPanel(List moduleTemplates, boolean processUnallocatedSpace) { + for (IngestModuleTemplate moduleTemplate : moduleTemplates) { + modules.add(new IngestModuleModel(moduleTemplate)); + } + this.processUnallocatedSpace = processUnallocatedSpace; initComponents(); customizeComponents(); } - - public void setContext(String context) { - this.context = context; + + List getIngestModuleTemplates() { + List moduleTemplates = new ArrayList<>(); + for (IngestModuleModel module : modules) { + IngestModuleTemplate moduleTemplate = module.getIngestModuleTemplate(); + if (module.hasModuleSettingsPanel()) { + IngestModuleSettings settings = module.getModuleSettingsPanel().getSettings(); + moduleTemplate.setModuleSettings(settings); + } + moduleTemplates.add(moduleTemplate); + } + return moduleTemplates; } - - - public IngestModuleAbstract getCurrentIngestModule() { - return currentModule; - } - - public List getModulesToStart() { - return tableModel.getSelectedModules(); - } - - public List getDisabledModules() { - return tableModel.getUnSelectedModules(); - } - - public boolean processUnallocSpaceEnabled() { + + boolean getProcessUnallocSpace() { return processUnallocCheckbox.isSelected(); } private void customizeComponents() { - modulesTable.setModel(tableModel); + modulesTable.setModel(new IngestModulesTableModel()); modulesTable.setTableHeader(null); modulesTable.setSelectionMode(ListSelectionModel.SINGLE_SELECTION); - //custom renderer for tooltips - ModulesTableRenderer renderer = new ModulesTableRenderer(); - - //customize column witdhs - final int width = modulesScrollPane.getPreferredSize().width; - TableColumn column = null; - for (int i = 0; i < modulesTable.getColumnCount(); i++) { - column = modulesTable.getColumnModel().getColumn(i); - if (i == 0) { + // Set the column widths in the table model and add a custom cell + // renderer that will display module descriptions from the module models + // as tooltips. + IngestModulesTableRenderer renderer = new IngestModulesTableRenderer(); + int width = modulesScrollPane.getPreferredSize().width; + for (int i = 0; i < modulesTable.getColumnCount(); ++i) { + TableColumn column = modulesTable.getColumnModel().getColumn(i); + if (0 == i) { column.setPreferredWidth(((int) (width * 0.15))); } else { column.setCellRenderer(renderer); @@ -98,35 +90,28 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; } } + // Add a selection listener to the table model that will display the + // ingest job options panel of the currently selected module model and + // enable or disable the resources configuration panel invocation button. modulesTable.getSelectionModel().addListSelectionListener(new ListSelectionListener() { @Override public void valueChanged(ListSelectionEvent e) { ListSelectionModel listSelectionModel = (ListSelectionModel) e.getSource(); if (!listSelectionModel.isSelectionEmpty()) { int index = listSelectionModel.getMinSelectionIndex(); - currentModule = tableModel.getModule(index); - - // add the module-specific configuration panel, if there is one + selectedModule = modules.get(index); simplePanel.removeAll(); - if (currentModule.hasSimpleConfiguration()) { - simplePanel.add(currentModule.getSimpleConfiguration(context)); + if (null != selectedModule.getModuleSettingsPanel()) { + simplePanel.add(selectedModule.getModuleSettingsPanel()); } simplePanel.revalidate(); simplePanel.repaint(); - advancedButton.setEnabled(currentModule.hasAdvancedConfiguration()); - } else { - currentModule = null; + advancedButton.setEnabled(null != selectedModule.getGlobalSettingsPanel()); } } }); - } - public void setProcessUnallocSpaceEnabled(final boolean enabled) { - processUnallocCheckbox.setSelected(enabled); - } - - public void setEnabledIngestModules(List enabledModules) { - tableModel.setSelectedModules(enabledModules); + processUnallocCheckbox.setSelected(processUnallocatedSpace); } /** @@ -172,7 +157,7 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; jPanel1.setBorder(javax.swing.BorderFactory.createLineBorder(new java.awt.Color(160, 160, 160))); jPanel1.setPreferredSize(new java.awt.Dimension(338, 257)); - advancedButton.setText(org.openide.util.NbBundle.getMessage(IngestDialogPanel.class, "IngestDialogPanel.advancedButton.text")); // NOI18N + advancedButton.setText(org.openide.util.NbBundle.getMessage(IngestJobConfigurationPanel.class, "IngestJobConfigurationPanel.advancedButton.text")); // NOI18N advancedButton.setEnabled(false); advancedButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -210,8 +195,8 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; processUnallocPanel.setBorder(javax.swing.BorderFactory.createLineBorder(new java.awt.Color(160, 160, 160))); - processUnallocCheckbox.setText(org.openide.util.NbBundle.getMessage(IngestDialogPanel.class, "IngestDialogPanel.processUnallocCheckbox.text")); // NOI18N - processUnallocCheckbox.setToolTipText(org.openide.util.NbBundle.getMessage(IngestDialogPanel.class, "IngestDialogPanel.processUnallocCheckbox.toolTipText")); // NOI18N + processUnallocCheckbox.setText(org.openide.util.NbBundle.getMessage(IngestJobConfigurationPanel.class, "IngestJobConfigurationPanel.processUnallocCheckbox.text")); // NOI18N + processUnallocCheckbox.setToolTipText(org.openide.util.NbBundle.getMessage(IngestJobConfigurationPanel.class, "IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText")); // NOI18N processUnallocCheckbox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { processUnallocCheckboxActionPerformed(evt); @@ -264,24 +249,29 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; private void advancedButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_advancedButtonActionPerformed final AdvancedConfigurationDialog dialog = new AdvancedConfigurationDialog(); + dialog.addApplyButtonListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { + if (selectedModule.hasGlobalSettingsPanel()) { + selectedModule.saveResourcesConfig(); + } dialog.close(); - currentModule.saveAdvancedConfiguration(); } }); + dialog.addWindowListener(new WindowAdapter() { @Override public void windowClosing(WindowEvent e) { dialog.close(); } }); - dialog.display(currentModule.getAdvancedConfiguration(context)); + + dialog.display(selectedModule.getGlobalSettingsPanel()); }//GEN-LAST:event_advancedButtonActionPerformed private void processUnallocCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_processUnallocCheckboxActionPerformed - // nothing to do here + processUnallocatedSpace = processUnallocCheckbox.isSelected(); }//GEN-LAST:event_processUnallocCheckboxActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton advancedButton; @@ -296,20 +286,77 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; private javax.swing.ButtonGroup timeGroup; // End of variables declaration//GEN-END:variables - private class ModulesTableModel extends AbstractTableModel { - - private List>moduleData = new ArrayList<>(); + /** + * A decorator for an ingest module template that adds ingest and global + * options panels with lifetimes equal to that of the ingest configuration + * panel. + */ + static private class IngestModuleModel { - public ModulesTableModel() { - List modules = IngestManager.getDefault().enumerateAllModules(); - for (IngestModuleAbstract ingestModuleAbstract : modules) { - moduleData.add(new AbstractMap.SimpleEntry<>(ingestModuleAbstract, Boolean.TRUE)); + private final IngestModuleTemplate moduleTemplate; + private IngestModuleGlobalSetttingsPanel globalSettingsPanel = null; + private IngestModuleSettingsPanel moduleSettingsPanel = null; + + IngestModuleModel(IngestModuleTemplate moduleTemplate) { + this.moduleTemplate = moduleTemplate; + if (moduleTemplate.hasModuleSettingsPanel()) { + moduleSettingsPanel = moduleTemplate.getModuleSettingsPanel(); + } + if (moduleTemplate.hasGlobalSettingsPanel()) { + globalSettingsPanel = moduleTemplate.getGlobalSettingsPanel(); } } + IngestModuleTemplate getIngestModuleTemplate() { + return moduleTemplate; + } + + String getName() { + return moduleTemplate.getModuleName(); + } + + String getDescription() { + return moduleTemplate.getModuleDescription(); + } + + void setEnabled(boolean enabled) { + moduleTemplate.setEnabled(enabled); + } + + boolean isEnabled() { + return moduleTemplate.isEnabled(); + } + + boolean hasModuleSettingsPanel() { + return moduleTemplate.hasModuleSettingsPanel(); + } + + IngestModuleSettingsPanel getModuleSettingsPanel() { + return moduleSettingsPanel; + } + + boolean hasGlobalSettingsPanel() { + return moduleTemplate.hasGlobalSettingsPanel(); + } + + IngestModuleGlobalSetttingsPanel getGlobalSettingsPanel() { + return globalSettingsPanel; + } + + void saveResourcesConfig() { + globalSettingsPanel.saveSettings(); + } + } + + /** + * Custom table model to display ingest module names and enable/disable + * ingest modules. + */ + private class IngestModulesTableModel extends AbstractTableModel { + @Override public int getRowCount() { - return moduleData.size(); + return modules.size(); } @Override @@ -319,11 +366,11 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; @Override public Object getValueAt(int rowIndex, int columnIndex) { - Map.Entry entry = moduleData.get(rowIndex); + IngestModuleModel module = modules.get(rowIndex); if (columnIndex == 0) { - return entry.getValue(); + return module.isEnabled(); } else { - return entry.getKey().getName(); + return module.getName(); } } @@ -335,7 +382,7 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; @Override public void setValueAt(Object aValue, int rowIndex, int columnIndex) { if (columnIndex == 0) { - moduleData.get(rowIndex).setValue((Boolean)aValue); + modules.get(rowIndex).setEnabled((boolean) aValue); } } @@ -343,103 +390,26 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; public Class getColumnClass(int c) { return getValueAt(0, c).getClass(); } - - public List getSelectedModules() { - List selectedModules = new ArrayList<>(); - for (Map.Entry entry : moduleData) { - if (entry.getValue().booleanValue()) { - selectedModules.add(entry.getKey()); - } - } - return selectedModules; - } - - public List getUnSelectedModules() { - List unselectedModules = new ArrayList<>(); - for (Map.Entry entry : moduleData) { - if (!entry.getValue().booleanValue()) { - unselectedModules.add(entry.getKey()); - } - } - return unselectedModules; - } - - /** - * Sets the given modules as selected in the modules table - * @param selectedModules - */ - public void setSelectedModules(List selectedModules) { - // unselect all modules - for (Map.Entry entry : moduleData) { - entry.setValue(Boolean.FALSE); - } - - // select only the given modules - for (IngestModuleAbstract selectedModule : selectedModules) { - getEntryForModule(selectedModule).setValue(Boolean.TRUE); - } - - // tell everyone about it - fireTableDataChanged(); - } - - /** - * Sets the given modules as NOT selected in the modules table - * @param selectedModules - */ - public void setUnselectedModules(List unselectedModules) { - // select all modules - for (Map.Entry entry : moduleData) { - entry.setValue(Boolean.TRUE); - } - - // unselect only the given modules - for (IngestModuleAbstract unselectedModule : unselectedModules) { - getEntryForModule(unselectedModule).setValue(Boolean.FALSE); - } - - // tell everyone about it - fireTableDataChanged(); - } - - public IngestModuleAbstract getModule(int row) { - return moduleData.get(row).getKey(); - } - - private Map.Entry getEntryForModule(IngestModuleAbstract module) { - Map.Entry entry = null; - for (Map.Entry anEntry : moduleData) { - if (anEntry.getKey().equals(module)) { - entry = anEntry; - break; - } - } - return entry; - } } /** - * Custom cell renderer for tool tips with module description + * Custom cell renderer to create tool tips displaying ingest module + * descriptions. */ - private class ModulesTableRenderer extends DefaultTableCellRenderer { - + private class IngestModulesTableRenderer extends DefaultTableCellRenderer { + List tooltips = new ArrayList<>(); - public ModulesTableRenderer() { - List modules = IngestManager.getDefault().enumerateAllModules(); - for (IngestModuleAbstract ingestModuleAbstract : modules) { - tooltips.add(ingestModuleAbstract.getDescription()); + public IngestModulesTableRenderer() { + for (IngestModuleModel moduleTemplate : modules) { + tooltips.add(moduleTemplate.getDescription()); } } @Override - public Component getTableCellRendererComponent( - JTable table, Object value, - boolean isSelected, boolean hasFocus, - int row, int column) { + public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column); - - if (column == 1) { + if (1 == column) { setToolTipText(tooltips.get(row)); } return this; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java new file mode 100755 index 0000000000..7984370d6f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -0,0 +1,66 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.List; +import org.sleuthkit.datamodel.AbstractFile; + +/** + * Provides an instance of an ingest module with services specific to the ingest + * job and the ingest pipeline of which the module is a part. + */ +public final class IngestJobContext { + + private final IngestJob ingestJob; + + IngestJobContext(IngestJob ingestJob) { + this.ingestJob = ingestJob; + } + + /** + * Gets the identifier of the ingest job associated with this context. + * + * @return The ingest job identifier. + */ + public long getJobId() { + return this.ingestJob.getId(); + } + + /** + * Determines whether the ingest job associated with the current context has + * been canceled. + * + * @return True if the job has been canceled, false otherwise. + */ + public boolean isJobCancelled() { + return this.ingestJob.isCancelled(); + } + + /** + * Adds one or more files to the files to be passed through the file ingest + * pipeline of the ingest job associated with the current context. + * + * @param files The files to be processed by the file ingest pipeline. + */ + public void addFilesToPipeline(List files) { + for (AbstractFile file : files) { + IngestManager.getDefault().scheduleFile(ingestJob.getId(), file); + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java new file mode 100644 index 0000000000..261647319b --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java @@ -0,0 +1,238 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import javax.swing.JPanel; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; +import org.sleuthkit.datamodel.Content; + +/** + * Provides a mechanism for creating and persisting an ingest job configuration + * for a particular context and for launching ingest jobs that process one or + * more data sources using the ingest job configuration. + */ +public final class IngestJobLauncher { + + private static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; + private static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; + private static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; + private final String launcherContext; + private final List warnings = new ArrayList<>(); + private IngestJobConfigurationPanel ingestConfigPanel; + + /** + * Constructs an ingest job launcher that creates and persists an ingest job + * configuration for a particular context and launches ingest jobs that + * process one or more data sources using the ingest job configuration. + * + * @param launcherContext The context identifier. + */ + public IngestJobLauncher(String launcherContext) { + this.launcherContext = launcherContext; + + // Get the ingest module factories discovered by the ingest module + // loader. + List moduleFactories = IngestModuleFactoryLoader.getInstance().getIngestModuleFactories(); + HashSet loadedModuleNames = new HashSet<>(); + for (IngestModuleFactory moduleFactory : moduleFactories) { + loadedModuleNames.add(moduleFactory.getModuleDisplayName()); + } + + // Get the enabled and disabled ingest modules settings for the current + // context. Observe that the default settings make all loaded ingest + // modules enabled. + HashSet enabledModuleNames = getModulesNamesFromSetting(ENABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(loadedModuleNames)); + HashSet disabledModuleNames = getModulesNamesFromSetting(DISABLED_INGEST_MODULES_KEY, ""); + + // Check for missing modules. + List missingModuleNames = new ArrayList<>(); + for (String moduleName : enabledModuleNames) { + if (!loadedModuleNames.contains(moduleName)) { + missingModuleNames.add(moduleName); + } + } + for (String moduleName : disabledModuleNames) { + if (!loadedModuleNames.contains(moduleName)) { + missingModuleNames.add(moduleName); + } + } + for (String moduleName : missingModuleNames) { + enabledModuleNames.remove(moduleName); + disabledModuleNames.remove(moduleName); + warnings.add(String.format("Previously loaded %s module could not be found", moduleName)); + } + + // Create ingest module templates. + List moduleTemplates = new ArrayList<>(); + for (IngestModuleFactory moduleFactory : moduleFactories) { + // NOTE: In the future, this code will be modified to get the + // module settings for the current context, if available, from + // storage; for now always use the defaults. + IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, moduleFactory.getDefaultModuleSettings()); + String moduleName = moduleTemplate.getModuleName(); + if (enabledModuleNames.contains(moduleName)) { + moduleTemplate.setEnabled(true); + } else if (disabledModuleNames.contains(moduleName)) { + moduleTemplate.setEnabled(true); + } else { + // The module factory was loaded, but the module name does not + // appear in the enabled/disabled module settings. Treat the + // module as a new module and enable it by default. + moduleTemplate.setEnabled(true); + enabledModuleNames.add(moduleName); + } + moduleTemplates.add(moduleTemplate); + } + + // Update the enabled/disabled ingest module settings to reflect any + // missing modules or newly discovered modules. + ModuleSettings.setConfigSetting(launcherContext, ENABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(enabledModuleNames)); + ModuleSettings.setConfigSetting(launcherContext, DISABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(disabledModuleNames)); + + // Get the process unallocated space flag setting. If the setting does + // not exist yet, default it to false. + if (ModuleSettings.settingExists(launcherContext, PARSE_UNALLOC_SPACE_KEY) == false) { + ModuleSettings.setConfigSetting(launcherContext, PARSE_UNALLOC_SPACE_KEY, "false"); + } + boolean processUnallocatedSpace = Boolean.parseBoolean(ModuleSettings.getConfigSetting(launcherContext, PARSE_UNALLOC_SPACE_KEY)); + + // Make the configuration panel for the context. + ingestConfigPanel = new IngestJobConfigurationPanel(moduleTemplates, processUnallocatedSpace); + } + + private HashSet getModulesNamesFromSetting(String key, String defaultSetting) { + // Get the ingest modules setting from the user's config file. + // If there is no such setting yet, create the default setting. + if (ModuleSettings.settingExists(launcherContext, key) == false) { + ModuleSettings.setConfigSetting(launcherContext, key, defaultSetting); + } + HashSet moduleNames = new HashSet<>(); + String modulesSetting = ModuleSettings.getConfigSetting(launcherContext, key); + if (!modulesSetting.isEmpty()) { + String[] settingNames = modulesSetting.split(", "); + for (String name : settingNames) { + // Map some old core module names to the current core module names. + switch (name) { + case "Thunderbird Parser": + case "MBox Parser": + moduleNames.add("Email Parser"); + break; + case "File Extension Mismatch Detection": + moduleNames.add("Extension Mismatch Detector"); + break; + case "EWF Verify": + case "E01 Verify": + moduleNames.add("E01 Verifier"); + break; + default: + moduleNames.add(name); + } + } + } + return moduleNames; + } + + /** + * Gets any warnings generated when the persisted ingest job configuration + * for the specified context is retrieved and loaded. + * + * @return A collection of warning messages. + */ + public List getIngestJobConfigWarnings() { + return warnings; + } + + /** + * Gets the user interface panel the launcher uses to obtain the user's + * ingest job configuration for the specified context. + * + * @return A JPanel with components that can be used to create an ingest job + * configuration. + */ + public JPanel getIngestJobConfigPanel() { + return ingestConfigPanel; + } + + /** + * Persists the ingest job configuration for the specified context. + */ + public void saveIngestJobConfig() { + List moduleTemplates = ingestConfigPanel.getIngestModuleTemplates(); + + // Save the enabled/disabled ingest module settings for the current context. + HashSet enabledModuleNames = new HashSet<>(); + HashSet disabledModuleNames = new HashSet<>(); + for (IngestModuleTemplate moduleTemplate : moduleTemplates) { + String moduleName = moduleTemplate.getModuleName(); + if (moduleTemplate.isEnabled()) { + enabledModuleNames.add(moduleName); + } else { + disabledModuleNames.add(moduleName); + } + } + ModuleSettings.setConfigSetting(launcherContext, ENABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(enabledModuleNames)); + ModuleSettings.setConfigSetting(launcherContext, DISABLED_INGEST_MODULES_KEY, makeCommaSeparatedList(disabledModuleNames)); + + // Save the process unallocated space setting for the current context. + String processUnalloc = Boolean.toString(ingestConfigPanel.getProcessUnallocSpace()); + ModuleSettings.setConfigSetting(launcherContext, PARSE_UNALLOC_SPACE_KEY, processUnalloc); + + // NOTE: In the future, this code will be modified to persist the ingest + // options for each ingest module for the current launch context. + } + + private static String makeCommaSeparatedList(HashSet input) { + if (input == null || input.isEmpty()) { + return ""; + } + + ArrayList list = new ArrayList<>(); + list.addAll(input); + StringBuilder csvList = new StringBuilder(); + for (int i = 0; i < list.size() - 1; ++i) { + csvList.append(list.get(i)).append(", "); + } + csvList.append(list.get(list.size() - 1)); + return csvList.toString(); + } + + /** + * Launches ingest jobs for one or more data sources using the ingest job + * configuration for the selected context. + * + * @param dataSources The data sources to ingest. + */ + public void startIngestJobs(List dataSources) { + // Filter out the disabled ingest module templates. + List enabledModuleTemplates = new ArrayList<>(); + List moduleTemplates = ingestConfigPanel.getIngestModuleTemplates(); + for (IngestModuleTemplate moduleTemplate : moduleTemplates) { + if (moduleTemplate.isEnabled()) { + enabledModuleTemplates.add(moduleTemplate); + } + } + + if ((!enabledModuleTemplates.isEmpty()) && (dataSources != null) && (!dataSources.isEmpty())) { + IngestManager.getDefault().scheduleDataSourceTasks(dataSources, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace()); + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index b8bbf43557..dd9222fba7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2013 Basis Technology Corp. + * Copyright 2013-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,21 +18,13 @@ */ package org.sleuthkit.autopsy.ingest; -import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.logging.Level; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.SwingWorker; @@ -40,147 +32,78 @@ import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandleFactory; import org.openide.util.Cancellable; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.coreutils.StopWatch; -import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; -import org.sleuthkit.autopsy.ingest.IngestScheduler.FileScheduler.FileTask; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstract.IngestModuleException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** - * IngestManager sets up and manages ingest modules runs them in a background - * thread notifies modules when work is complete or should be interrupted - * processes messages from modules via messenger proxy and posts them to GUI. - * - * This runs as a singleton and you can access it using the getDefault() method. - * + * Manages the execution of ingest jobs. */ public class IngestManager { private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); - private IngestManagerStats stats; - private boolean processUnallocSpace = true; - //queues + private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); + private static IngestManager instance; private final IngestScheduler scheduler; - //workers - private IngestAbstractFileProcessor abstractFileIngester; - private List dataSourceIngesters; - private SwingWorker queueWorker; - //modules - private List dataSourceModules; - private List abstractFileModules; - // module return values - private final Map abstractFileModulesRetValues = new HashMap(); - //notifications - private final static PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); - //monitor private final IngestMonitor ingestMonitor = new IngestMonitor(); - //module loader - private IngestModuleLoader moduleLoader = null; - //property file name id for the module - public final static String MODULE_PROPERTIES = NbBundle.getMessage(IngestManager.class, - "IngestManager.moduleProperties.text"); - private volatile int messageID = 0; + private final HashMap ingestJobs = new HashMap<>(); + private TaskSchedulingWorker taskSchedulingWorker; + private FileTaskWorker fileTaskWorker; + private DataSourceTaskWorker dataSourceTaskWorker; + private long nextDataSourceTaskId = 0; + private long nextThreadId = 0; + private volatile IngestUI ingestMessageBox; /** - * Possible events about ingest modules Event listeners can get the event - * name by using String returned by toString() method on the specific event. + * Ingest events. */ - public enum IngestModuleEvent { + public enum IngestEvent { + // RJCTODO: Update comments /** - * Event sent when an ingest module has been started. Second - * argument of the property change is a string form of the module name - * and the third argument is null. + * Event sent when an ingest module has been started. Second argument of + * the property change is a string form of the module name and the third + * argument is null. */ STARTED, - /** - * Event sent when an ingest module has completed processing by its own - * means. Second - * argument of the property change is a string form of the module name - * and the third argument is null. + * Event sent when an ingest module has completed processing by its own + * means. Second argument of the property change is a string form of the + * module name and the third argument is null. * * This event is generally used by listeners to perform a final data * view refresh (listeners need to query all data from the blackboard). */ COMPLETED, - /** * Event sent when an ingest module has stopped processing, and likely * not all data has been processed. Second argument of the property - * change is a string form of the module name and third argument is null. + * change is a string form of the module name and third argument is + * null. */ STOPPED, - /** - * Event sent when ingest module posts new data to blackboard or somewhere - * else. Second argument of the - * property change fired contains ModuleDataEvent object and third - * argument is null. The object can contain encapsulated new data - * created by the module. Listener can also query new data as needed. + * Event sent when ingest module posts new data to blackboard or + * somewhere else. Second argument of the property change fired contains + * ModuleDataEvent object and third argument is null. The object can + * contain encapsulated new data created by the module. Listener can + * also query new data as needed. */ DATA, - /** * Event send when content changed, either its attributes changed, or - * new content children have been added. I.e. from ZIP files or Carved files + * new content children have been added. I.e. from ZIP files or Carved + * files */ CONTENT_CHANGED, - - /** - * Event sent when a file has finished going through a pipeline of modules. - * Second argument is the object ID. Third argument is null + * Event sent when a file has finished going through a pipeline of + * modules. Second argument is the object ID. Third argument is null */ FILE_DONE, - }; - //ui - //Initialized by Installer in AWT thread once the Window System is ready - private volatile IngestUI ui; // = null; //IngestMessageTopComponent.findInstance(); - //singleton - private static volatile IngestManager instance; private IngestManager() { - dataSourceIngesters = new ArrayList(); - - scheduler = IngestScheduler.getInstance(); - - //setup current modules and listeners for modules changes - initModules(); - - } - - private void initModules() { - try { - moduleLoader = IngestModuleLoader.getDefault(); - abstractFileModules = moduleLoader.getAbstractFileIngestModules(); - - moduleLoader.addModulesReloadedListener(new PropertyChangeListener() { - @Override - public void propertyChange(PropertyChangeEvent evt) { - if (evt.getPropertyName().equals(IngestModuleLoader.Event.ModulesReloaded.toString())) { - //TODO might need to not allow to remove modules if they are running - abstractFileModules = moduleLoader.getAbstractFileIngestModules(); - dataSourceModules = moduleLoader.getDataSourceIngestModules(); - } - } - }); - dataSourceModules = moduleLoader.getDataSourceIngestModules(); - } catch (IngestModuleLoaderException ex) { - logger.log(Level.SEVERE, "Error getting module loader"); - } - } - - /** - * called by Installer in AWT thread once the Window System is ready - */ - void initUI() { - if (this.ui == null) { - this.ui = IngestMessageTopComponent.findInstance(); - } + this.scheduler = IngestScheduler.getInstance(); } /** @@ -188,562 +111,251 @@ public class IngestManager { * * @returns Instance of class. */ - public static IngestManager getDefault() { + synchronized public static IngestManager getDefault() { if (instance == null) { - synchronized (IngestManager.class) { - if (instance == null) { - logger.log(Level.INFO, "creating manager instance"); - instance = new IngestManager(); - } - } + instance = new IngestManager(); } return instance; } /** - * Add property change listener to listen to ingest events as defined in IngestModuleEvent. - * - * @param l PropertyChangeListener to register + * called by Installer in AWT thread once the Window System is ready */ - public static synchronized void addPropertyChangeListener(final PropertyChangeListener l) { - pcs.addPropertyChangeListener(l); + void initIngestMessageInbox() { + if (this.ingestMessageBox == null) { + this.ingestMessageBox = IngestMessageTopComponent.findInstance(); + } + } + + synchronized private long getNextDataSourceTaskId() { + return ++this.nextDataSourceTaskId; + } + + synchronized private long getNextThreadId() { + return ++this.nextThreadId; + } + + /** + * Add property change listener to listen to ingest events as defined in + * IngestModuleEvent. + * + * @param listener PropertyChangeListener to register + */ + public static synchronized void addPropertyChangeListener(final PropertyChangeListener listener) { + pcs.addPropertyChangeListener(listener); + } + + public static synchronized void removePropertyChangeListener(final PropertyChangeListener listener) { + pcs.removePropertyChangeListener(listener); } static synchronized void fireModuleEvent(String eventType, String moduleName) { try { pcs.firePropertyChange(eventType, moduleName, null); - } - catch (Exception e) { + } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); + NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), + MessageNotifyUtil.MessageType.ERROR); } } - - + /** * Fire event when file is done with a pipeline run + * * @param objId ID of file that is done */ static synchronized void fireFileDone(long objId) { try { - pcs.firePropertyChange(IngestModuleEvent.FILE_DONE.toString(), objId, null); - } - catch (Exception e) { + pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), objId, null); + } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); + NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), + MessageNotifyUtil.MessageType.ERROR); } } - /** - * Fire event for ModuleDataEvent (when modules post data to blackboard, etc.) - * @param moduleDataEvent + * Fire event for ModuleDataEvent (when modules post data to blackboard, + * etc.) + * + * @param moduleDataEvent */ static synchronized void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { try { - pcs.firePropertyChange(IngestModuleEvent.DATA.toString(), moduleDataEvent, null); - } - catch (Exception e) { + pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null); + } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); + NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), + MessageNotifyUtil.MessageType.ERROR); } } /** - * Fire event for ModuleContentChanged (when modules create new content that needs to be analyzed) - * @param moduleContentEvent + * Fire event for ModuleContentChanged (when modules create new content that + * needs to be analyzed) + * + * @param moduleContentEvent */ static synchronized void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { try { - pcs.firePropertyChange(IngestModuleEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null); - } - catch (Exception e) { + pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null); + } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); + NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), + MessageNotifyUtil.MessageType.ERROR); } } /** - * Returns the return value from a previously run module on the file being - * currently analyzed. - * - * @param moduleName Name of module. - * @returns Return value from that module if it was previously run. - */ - IngestModuleAbstractFile.ProcessResult getAbstractFileModuleResult(String moduleName) { - synchronized (abstractFileModulesRetValues) { - if (abstractFileModulesRetValues.containsKey(moduleName)) { - return abstractFileModulesRetValues.get(moduleName); - } else { - return IngestModuleAbstractFile.ProcessResult.UNKNOWN; - } - } - } - - /** - * Multiple data-sources version of scheduleDataSource() method. Enqueues multiple sources inputs (Content objects) - * and associated modules at once + * Multiple data-sources version of scheduleDataSource() method. Enqueues + * multiple sources inputs (Content objects) and associated modules at once * * @param modules modules to scheduleDataSource on every data source - * @param inputs input data sources to enqueue and scheduleDataSource the ingest modules on + * @param inputs input data sources to enqueue and scheduleDataSource the + * ingest modules on */ - public void scheduleDataSource(final List modules, final List inputs) { - logger.log(Level.INFO, "Will enqueue number of inputs: " + inputs.size() - + " to " + modules.size() + " modules."); - - if (!isIngestRunning() && ui != null) { - ui.clearMessages(); + void scheduleDataSourceTasks(final List dataSources, final List moduleTemplates, boolean processUnallocatedSpace) { + if (!isIngestRunning() && ingestMessageBox != null) { + ingestMessageBox.clearMessages(); } - queueWorker = new EnqueueWorker(modules, inputs); - queueWorker.execute(); + taskSchedulingWorker = new TaskSchedulingWorker(dataSources, moduleTemplates, processUnallocatedSpace); + taskSchedulingWorker.execute(); - if (ui != null) { - ui.restoreMessages(); + if (ingestMessageBox != null) { + ingestMessageBox.restoreMessages(); } } /** - * IngestManager entry point, enqueues data to be processed and starts new ingest - * as needed, or just enqueues data to an existing pipeline. - * - * Spawns - * background thread which enumerates all sorted files and executes chosen - * modules per file in a pre-determined order. Notifies modules when work is - * complete or should be interrupted using complete() and stop() calls. Does - * not block and can be called multiple times to enqueue more work to - * already running background ingest process. + * IngestManager entry point, enqueues data to be processed and starts new + * ingest as needed, or just enqueues data to an existing pipeline. + * + * Spawns background thread which enumerates all sorted files and executes + * chosen modules per file in a pre-determined order. Notifies modules when + * work is complete or should be interrupted using complete() and stop() + * calls. Does not block and can be called multiple times to enqueue more + * work to already running background ingest process. * * @param modules modules to scheduleDataSource on the data source input - * @param input input data source Content objects to scheduleDataSource the ingest modules on + * @param input input data source Content objects to scheduleDataSource the + * ingest modules on */ - public void scheduleDataSource(final List modules, final Content input) { - List inputs = new ArrayList(); - inputs.add(input); - logger.log(Level.INFO, "Will enqueue input: " + input.getName()); - scheduleDataSource(modules, inputs); + void scheduleDataSourceTask(final Content dataSource, final List moduleTemplates, boolean processUnallocatedSpace) { + List dataSources = new ArrayList<>(); + dataSources.add(dataSource); + scheduleDataSourceTasks(dataSources, moduleTemplates, processUnallocatedSpace); } /** - * Schedule a file for ingest and add it to ongoing file ingest process on the same data source. - * Scheduler updates the current progress. + * Schedule a file for ingest and add it to ongoing file ingest process on + * the same data source. Scheduler updates the current progress. * - * The file to be added is usually a product of a currently ran ingest. - * Now we want to process this new file with the same ingest context. + * The file to be added is usually a product of a currently ran ingest. Now + * we want to process this new file with the same ingest context. * * @param file file to be scheduled * @param pipelineContext ingest context used to ingest parent of the file * to be scheduled */ - void scheduleFile(AbstractFile file, PipelineContext pipelineContext) { - scheduler.getFileScheduler().schedule(file, pipelineContext); + void scheduleFile(long ingestJobId, AbstractFile file) { + IngestJob job = this.ingestJobs.get(ingestJobId); + if (job == null) { + logger.log(Level.SEVERE, "Unable to map ingest job id (id = {0}) to an ingest job, failed to schedule file (id = {1})", new Object[]{ingestJobId, file.getId()}); + MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), + "Unable to associate " + file.getName() + " with ingest job, file will not be processed by ingest nodules", + MessageNotifyUtil.MessageType.ERROR); + } + + scheduler.getFileScheduler().scheduleFile(job, file); } /** - * Starts the File-level Ingest Module pipeline and the Data Source-level Ingest Modules - * for the queued up data sources and files. + * Starts the File-level Ingest Module pipeline and the Data Source-level + * Ingest Modules for the queued up data sources and files. * * if AbstractFile module is still running, do nothing and allow it to * consume queue otherwise start /restart AbstractFile worker * - * data source ingest workers run per (module,content). Checks if one for the same (module,content) - * is already running otherwise start/restart the worker + * data source ingest workers run per (module,content). Checks if one for + * the same (module,content) is already running otherwise start/restart the + * worker */ private synchronized void startAll() { - final IngestScheduler.DataSourceScheduler dataSourceScheduler = scheduler.getDataSourceScheduler(); - final IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler(); - boolean allInited = true; - IngestModuleAbstract failedModule = null; - String errorMessage = ""; - logger.log(Level.INFO, "DataSource queue: " + dataSourceScheduler.toString()); - logger.log(Level.INFO, "File queue: " + fileScheduler.toString()); - if (!ingestMonitor.isRunning()) { ingestMonitor.start(); } - ///////// - // Start the data source-level ingest modules - List newThreads = new ArrayList<>(); - - // cycle through each data source content in the queue - while (dataSourceScheduler.hasNext()) { - if (allInited == false) { - break; - } - //dequeue - // get next data source content and set of modules - final DataSourceTask dataSourceTask = dataSourceScheduler.next(); - - // check if each module for this data source content is already running - for (IngestModuleDataSource dataSourceTaskModule : dataSourceTask.getModules()) { - boolean alreadyRunning = false; - for (IngestDataSourceThread worker : dataSourceIngesters) { - // ignore threads that are on different data sources - if (!worker.getContent().equals(dataSourceTask.getContent())) { - continue; //check next worker - } - //same data source, check module (by name, not id, since different instances) - if (worker.getModule().getName().equals(dataSourceTaskModule.getName())) { - alreadyRunning = true; - logger.log(Level.INFO, "Data Source Ingester <" + dataSourceTask.getContent() - + ", " + dataSourceTaskModule.getName() + "> is already running"); - break; - } - } - //checked all workers - if (alreadyRunning == false) { - logger.log(Level.INFO, "Starting new data source Ingester <" + dataSourceTask.getContent() - + ", " + dataSourceTaskModule.getName() + ">"); - //data source modules are now initialized per instance - - IngestModuleInit moduleInit = new IngestModuleInit(); - - PipelineContext dataSourcepipelineContext = - dataSourceTask.getPipelineContext(); - - final IngestDataSourceThread newDataSourceWorker = new IngestDataSourceThread(this, - dataSourcepipelineContext, dataSourceTask.getContent(), dataSourceTaskModule, moduleInit); - try { - newDataSourceWorker.init(); - } catch (Exception e) { - logger.log(Level.SEVERE, "DataSource ingest module failed init(): " + dataSourceTaskModule.getName(), e); - allInited = false; - failedModule = dataSourceTaskModule; - errorMessage = e.getMessage(); - break; - } - dataSourceIngesters.add(newDataSourceWorker); - // Add the worker to the list of new IngestThreads to be started - // if all modules initialize. - newThreads.add(newDataSourceWorker); - } + if (scheduler.getDataSourceScheduler().hasNext()) { + if (dataSourceTaskWorker == null || dataSourceTaskWorker.isDone()) { + dataSourceTaskWorker = new DataSourceTaskWorker(getNextThreadId()); + dataSourceTaskWorker.execute(); } } - - // Check to make sure all modules initialized - if (allInited == false) { - displayInitError(failedModule.getName(), errorMessage); - dataSourceIngesters.removeAll(newThreads); - return; - } - //AbstractFile ingester - boolean startAbstractFileIngester = false; - if (fileScheduler.hasNext()) { - if (abstractFileIngester == null) { - startAbstractFileIngester = true; - logger.log(Level.INFO, "Starting initial AbstractFile ingester"); - } - //if worker had completed, restart it in case data is still enqueued - else if (abstractFileIngester.isDone()) { - startAbstractFileIngester = true; - logger.log(Level.INFO, "Restarting AbstractFile ingester"); + if (scheduler.getFileScheduler().hasNext()) { + if (fileTaskWorker == null || fileTaskWorker.isDone()) { + fileTaskWorker = new FileTaskWorker(getNextThreadId()); + fileTaskWorker.execute(); } - } else { - logger.log(Level.INFO, "no new AbstractFile enqueued, no ingester needed"); - } - - if (startAbstractFileIngester) { - stats = new IngestManagerStats(); - abstractFileIngester = new IngestAbstractFileProcessor(); - //init all fs modules, everytime new worker starts - - for (IngestModuleAbstractFile s : abstractFileModules) { - // This was added at one point to remove the message about non-configured HashDB even - // when HashDB was not enabled. However, it adds some problems if a second ingest is - // kicked off whiel the first is ongoing. If the 2nd ingest has a module enabled that - // was not initially enabled, it will never have init called. We also need to call - // complete and need a similar way of passing down data to that thread to tell it which - // it shoudl call complete on (otherwise it could call complete on a module that never - // had init() called. - //if (fileScheduler.hasModuleEnqueued(s) == false) { - // continue; - //} - IngestModuleInit moduleInit = new IngestModuleInit(); - try { - s.init(moduleInit); - } catch (IngestModuleException e) { - logger.log(Level.SEVERE, "File ingest module failed init(): " + s.getName(), e); - allInited = false; - failedModule = s; - errorMessage = e.getMessage(); - break; - } - } - } - - if (allInited) { - // Start DataSourceIngestModules - for (IngestDataSourceThread dataSourceWorker : newThreads) { - dataSourceWorker.execute(); - IngestManager.fireModuleEvent(IngestModuleEvent.STARTED.toString(), dataSourceWorker.getModule().getName()); - } - // Start AbstractFileIngestModules - if (startAbstractFileIngester) { - abstractFileIngester.execute(); - } - } else { - displayInitError(failedModule.getName(), errorMessage); - dataSourceIngesters.removeAll(newThreads); - abstractFileIngester = null; } } - - /** - * Open a dialog box to report an initialization error to the user. - * - * @param moduleName The name of the module that failed to initialize. - * @param errorMessage The message gotten from the exception that was thrown. - */ - private void displayInitError(String moduleName, String errorMessage) { - MessageNotifyUtil.Message.error( - NbBundle.getMessage(this.getClass(), "IngestManager.displayInitError.failedToLoad.msg", moduleName, - errorMessage)); + + synchronized void reportThreadDone(long threadId) { + for (IngestJob job : ingestJobs.values()) { + job.releaseIngestPipelinesForThread(threadId); + if (job.areIngestPipelinesShutDown()) { + ingestJobs.remove(job.getId()); + } + } } - /** - * stop currently running threads if any (e.g. when changing a case) - */ synchronized void stopAll() { - //stop queue worker - if (queueWorker != null) { - queueWorker.cancel(true); - queueWorker = null; + // First get the task scheduling worker to stop. + if (taskSchedulingWorker != null) { + taskSchedulingWorker.cancel(true); + while (!taskSchedulingWorker.isDone()) { + // Wait. + } + taskSchedulingWorker = null; } - //empty queues + // Now mark all of the ingest jobs as cancelled. This way the ingest + // modules will know they are being shut down due to cancellation when + // the ingest worker threads release their pipelines. + for (IngestJob job : ingestJobs.values()) { + job.cancel(); + } + + // Cancel the worker threads. + if (dataSourceTaskWorker != null) { + dataSourceTaskWorker.cancel(true); + } + if (fileTaskWorker != null) { + fileTaskWorker.cancel(true); + } + + // Jettision the remaining tasks. This will dispose of any tasks that + // the scheduling worker queued up before it was cancelled. scheduler.getFileScheduler().empty(); scheduler.getDataSourceScheduler().empty(); - - //stop module workers - if (abstractFileIngester != null) { - //send signals to all file modules - for (IngestModuleAbstractFile s : this.abstractFileModules) { - if (isModuleRunning(s)) { - try { - s.stop(); - } catch (Exception e) { - logger.log(Level.WARNING, "Unexpected exception while stopping module: " + s.getName(), e); - } - } - - } - //stop fs ingester thread - boolean cancelled = abstractFileIngester.cancel(true); - if (!cancelled) { - logger.log(Level.INFO, "Unable to cancel file ingest worker, likely already stopped"); - } - - abstractFileIngester = null; - - } - - List toStop = new ArrayList(); - toStop.addAll(dataSourceIngesters); - - for (IngestDataSourceThread dataSourceWorker : toStop) { - IngestModuleDataSource s = dataSourceWorker.getModule(); - - //stop the worker thread if thread is running - boolean cancelled = dataSourceWorker.cancel(true); - if (!cancelled) { - logger.log(Level.INFO, "Unable to cancel data source ingest worker for module: " - + dataSourceWorker.getModule().getName() + " data source: " + dataSourceWorker.getContent().getName()); - } - - //stop notification to module to cleanup resources - if (isModuleRunning(s)) { - try { - dataSourceWorker.getModule().stop(); - } catch (Exception e) { - logger.log(Level.WARNING, "Exception while stopping module: " + s.getName(), e); - } - } - } - - logger.log(Level.INFO, "stopped all"); } /** - * Test if any ingester modules are running + * Test if any ingest modules are running * * @return true if any module is running, false otherwise */ public synchronized boolean isIngestRunning() { - if (isEnqueueRunning()) { - return true; - } else if (isFileIngestRunning()) { - return true; - } else if (isDataSourceIngestRunning()) { - return true; - } else { - return false; - } - + return ((taskSchedulingWorker != null && !taskSchedulingWorker.isDone()) + || (fileTaskWorker != null && !fileTaskWorker.isDone()) + || (fileTaskWorker != null && !fileTaskWorker.isDone())); } - /** - * Test is any file ingest modules are running. - * - * @return true if any ingest modules are running, false otherwise - */ - public synchronized boolean areModulesRunning() { - for (IngestModuleAbstract serv : abstractFileModules) { - if (serv.hasBackgroundJobsRunning()) { - return true; - } - } - for (IngestDataSourceThread thread : dataSourceIngesters) { - if (isModuleRunning(thread.getModule())) { - return false; - } - } - return false; - } - - /** - * check if ingest is currently being enqueued - */ - public synchronized boolean isEnqueueRunning() { - if (queueWorker != null && !queueWorker.isDone()) { - return true; - } - return false; - } - - /** - * check if the file-level ingest pipeline is running - */ - public synchronized boolean isFileIngestRunning() { - if (abstractFileIngester != null && !abstractFileIngester.isDone()) { - return true; - } - return false; - } - - /** - * check the status of the data-source-level ingest pipeline - */ - public synchronized boolean isDataSourceIngestRunning() { - if (dataSourceIngesters.isEmpty()) { - return false; - } - - //in case there are still data source ingesters in the queue but already done - boolean allDone = true; - for (IngestDataSourceThread ii : dataSourceIngesters) { - if (ii.isDone() == false) { - allDone = false; - break; - } - } - if (allDone) { - return false; - } else { - return true; - } - } - - /** - * check if the module is running (was started and not yet complete/stopped) - * give a complete answer, i.e. it's already consumed all files but it might - * have background threads running - * - */ - public boolean isModuleRunning(final IngestModuleAbstract module) { - - if (module.getType() == IngestModuleAbstract.ModuleType.AbstractFile) { - IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler(); - - if (fileScheduler.hasModuleEnqueued((IngestModuleAbstractFile) module)) { - //has work enqueued, so running - return true; - } else { - //not in the queue, but could still have bkg work running - return module.hasBackgroundJobsRunning(); - } - - } else { - //data source module - synchronized (this) { - if (dataSourceIngesters.isEmpty()) { - return false; - } - IngestDataSourceThread imt = null; - for (IngestDataSourceThread ii : dataSourceIngesters) { - if (ii.getModule().equals(module)) { - imt = ii; - break; - } - } - - if (imt == null) { - return false; - } - - if (imt.isDone() == false) { - return true; - } else { - return false; - } - } - } - } - - /** - * Check if data source scheduler has files in queues - * @return true if more sources in queues, false otherwise - */ - public boolean getDataSourceSchedulerHasNext() { - return this.scheduler.getDataSourceScheduler().hasNext(); - } - - /** - * Check if file scheduler has files in queues - * @return true if more files in queues, false otherwise - */ - public boolean getFileSchedulerHasNext() { - return scheduler.getFileScheduler().hasNext(); - } - - - /** - * returns if manager is currently configured to process unalloc space - * - * @return true if process unaloc space is set - */ - boolean getProcessUnallocSpace() { - return processUnallocSpace; - } - - /** - * Sets process unalloc space setting on the manager - * - * @param processUnallocSpace - */ - public void setProcessUnallocSpace(boolean processUnallocSpace) { - this.processUnallocSpace = processUnallocSpace; - } - - /** - * returns ingest summary report (how many files ingested, any errors, etc) - */ - String getReport() { - return stats.toString(); - } - - - - /** * Module publishes message using InegestManager handle Does not block. The * message gets enqueued in the GUI thread and displayed in a widget @@ -751,16 +363,9 @@ public class IngestManager { * multiple times. Viewer will attempt to identify duplicate messages and * filter them out (slower) */ - void postMessage(final IngestMessage message) { - - if (stats != null) { - //record the error for stats, if stats are running - if (message.getMessageType() == MessageType.ERROR) { - stats.addError(message.getSource()); - } - } - if (ui != null) { - ui.displayMessage(message); + void postIngestMessage(IngestMessage message) { + if (ingestMessageBox != null) { + ingestMessageBox.displayMessage(message); } } @@ -779,555 +384,257 @@ public class IngestManager { } } - /** - * helper to return all loaded data-source ingest modules managed sorted in order as - * specified in pipeline_config XML - */ - public List enumerateDataSourceModules() { - return moduleLoader.getDataSourceIngestModules(); - } + private class TaskSchedulingWorker extends SwingWorker { - /** - * helper to return all loaded file modules managed sorted in order as - * specified in pipeline_config XML - */ - public List enumerateAbstractFileModules() { - return moduleLoader.getAbstractFileIngestModules(); - } - - public List enumerateAllModules() { - List modules = new ArrayList<>(); - modules.addAll(enumerateDataSourceModules()); - modules.addAll(enumerateAbstractFileModules()); - return modules; - } - - //data source worker to remove itself when complete or interrupted - void removeDataSourceIngestWorker(IngestDataSourceThread worker) { - //remove worker - synchronized (this) { - dataSourceIngesters.remove(worker); - } - } - - /** - * collects IngestManager statistics during runtime - */ - private class IngestManagerStats { - - private Date startTime; - private Date endTime; - private int errorsTotal; - private Map errors; - private final DateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private final StopWatch timer = new StopWatch(); - private IngestModuleAbstract currentModuleForTimer; - //file module timing stats, datasource module timers are logged in IngestDataSourceThread class - private final Map fileModuleTimers = new HashMap(); - - IngestManagerStats() { - errors = new HashMap(); - } - - /** - * records start time of the file process for the module must be - * followed by logFileModuleEndProcess for the same module - * - * @param module to record start time for processing a file - */ - void logFileModuleStartProcess(IngestModuleAbstract module) { - timer.reset(); - timer.start(); - currentModuleForTimer = module; - } - - /** - * records stop time of the file process for the module must be preceded - * by logFileModuleStartProcess for the same module - * - * @param module to record stop time for processing a file - */ - void logFileModuleEndProcess(IngestModuleAbstract module) { - timer.stop(); - if (module != currentModuleForTimer) { - logger.log(Level.WARNING, "Invalid module passed in to record stop processing: " + module.getName() - + ", expected: " + currentModuleForTimer.getName()); - } else { - final long elapsed = timer.getElapsedTime(); - final long current = fileModuleTimers.get(module.getName()); - fileModuleTimers.put(module.getName(), elapsed + current); - } - - currentModuleForTimer = null; - } - - String getFileModuleStats() { - StringBuilder sb = new StringBuilder(); - for (final String moduleName : fileModuleTimers.keySet()) { - sb.append(NbBundle.getMessage(this.getClass(), - "IngestManager.getFileModStats.moduleInfo.text", - moduleName, fileModuleTimers.get(moduleName) / 1000)); - } - return sb.toString(); - } - - @Override - public String toString() { - final String EOL = System.getProperty("line.separator"); - StringBuilder sb = new StringBuilder(); - if (startTime != null) { - sb.append(NbBundle.getMessage(this.getClass(), "IngestManager.toString.startTime.text", - dateFormatter.format(startTime), EOL)); - } - if (endTime != null) { - sb.append(NbBundle.getMessage(this.getClass(), "IngestManager.toString.endTime.text", - dateFormatter.format(endTime), EOL)); - } - sb.append(NbBundle.getMessage(this.getClass(), "IngestManager.toString.totalIngestTime.text", - getTotalTimeString(), EOL)); - sb.append(NbBundle.getMessage(this.getClass(), "IngestManager.toString.totalErrs.text", errorsTotal, EOL)); - if (errorsTotal > 0) { - sb.append(NbBundle.getMessage(this.getClass(), "IngestManager.toString.errsPerMod.text")); - for (String moduleName : errors.keySet()) { - sb.append("\t").append(moduleName).append(": ").append(errors.get(moduleName)).append(EOL); - } - } - return sb.toString(); - } - - public String toHtmlString() { - StringBuilder sb = new StringBuilder(); - sb.append(""); - sb.append(NbBundle.getMessage(this.getClass(), "IngestManager.toHtmlStr.ingestTime.text", - getTotalTimeString())) - .append("
"); - sb.append(NbBundle.getMessage(this.getClass(), "IngestManager.toHtmlStr.totalErrs.text", errorsTotal)) - .append("
"); - sb.append("\n"); - - for (final String moduleName : fileModuleTimers.keySet()) { - sb.append("\n"); - } - sb.append("
") - .append(NbBundle.getMessage(this.getClass(), "IngestManager.toHtmlStr.module.text")) - .append("") - .append(NbBundle.getMessage(this.getClass(), "IngestManager.toHtmlStr.time.text")) - .append("") - .append(NbBundle.getMessage(this.getClass(), "IngestManager.toHtmlStr.errors.text")) - .append("
").append(moduleName).append(""); - sb.append(msToString(fileModuleTimers.get(moduleName))).append(""); - if (errors.get(moduleName) == null) { - sb.append("0"); - } else { - sb.append(errors.get(moduleName)); - } - sb.append("
"); - sb.append(""); - return sb.toString(); - } - - void start() { - startTime = new Date(); - - for (IngestModuleAbstractFile module : abstractFileModules) { - fileModuleTimers.put(module.getName(), 0L); - } - } - - void end() { - endTime = new Date(); - } - - long getTotalTime() { - if (startTime == null || endTime == null) { - return 0; - } - return endTime.getTime() - startTime.getTime(); - } - - String getStartTimeString() { - return dateFormatter.format(startTime); - } - - String getEndTimeString() { - return dateFormatter.format(endTime); - } - - /** - * convert time in miliseconds to printable string in XX:YY:ZZ format. - * @param ms - * @return - */ - private String msToString(long ms) { - long hours = TimeUnit.MILLISECONDS.toHours(ms); - ms -= TimeUnit.HOURS.toMillis(hours); - long minutes = TimeUnit.MILLISECONDS.toMinutes(ms); - ms -= TimeUnit.MINUTES.toMillis(minutes); - long seconds = TimeUnit.MILLISECONDS.toSeconds(ms); - final StringBuilder sb = new StringBuilder(); - sb.append(hours < 10 ? "0" : "").append(hours).append(':').append(minutes < 10 ? "0" : "").append(minutes).append(':').append(seconds < 10 ? "0" : "").append(seconds); - return sb.toString(); - } - - String getTotalTimeString() { - long ms = getTotalTime(); - return msToString(ms); - } - - synchronized void addError(IngestModuleAbstract source) { - ++errorsTotal; - String moduleName = source.getName(); - Integer curModuleErrorI = errors.get(moduleName); - if (curModuleErrorI == null) { - errors.put(moduleName, 1); - } else { - errors.put(moduleName, curModuleErrorI + 1); - } - } - } - - /** - * File ingest pipeline processor. Worker thread that queries - * the scheduler for new files. - * Modules are assumed to already be initialized. - * runs until AbstractFile queue is - * consumed New instance is created and started when data arrives and - * previous pipeline completed. - */ - private class IngestAbstractFileProcessor extends SwingWorker { - - private Logger logger = Logger.getLogger(IngestAbstractFileProcessor.class.getName()); - //progress bar + private final List dataSources; + private final List moduleTemplates; + private final boolean processUnallocatedSpace; private ProgressHandle progress; + TaskSchedulingWorker(List dataSources, List moduleTemplates, boolean processUnallocatedSpace) { + this.dataSources = dataSources; + this.moduleTemplates = moduleTemplates; + this.processUnallocatedSpace = processUnallocatedSpace; + } + @Override protected Object doInBackground() throws Exception { - - logger.log(Level.INFO, "Starting background ingest file processor"); - logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); - - stats.start(); - - //notify main thread modules started - for (IngestModuleAbstractFile s : abstractFileModules) { - IngestManager.fireModuleEvent(IngestModuleEvent.STARTED.toString(), s.getName()); - } - - final String displayName = NbBundle - .getMessage(this.getClass(), "IngestManager.IngestAbstractFileProcessor.displayName"); - progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "Filed ingest cancelled by user."); - if (progress != null) { - progress.setDisplayName(NbBundle.getMessage(this.getClass(), - "IngestManager.IngestAbstractFileProcessor.process.cancelling", - displayName)); - } - return IngestAbstractFileProcessor.this.cancel(true); - } - }); - - final IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler(); - - //initialize the progress bar - progress.start(); - progress.switchToIndeterminate(); - //set initial totals and processed (to be updated as we process or new files are scheduled) - int totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); - progress.switchToDeterminate(totalEnqueuedFiles); - int processedFiles = 0; - - //process AbstractFiles queue - while (fileScheduler.hasNext()) { - final FileTask fileTask = fileScheduler.next(); - final DataSourceTask dataSourceTask = fileTask.getDataSourceTask(); - final PipelineContext filepipelineContext = dataSourceTask.getPipelineContext(); - - final AbstractFile fileToProcess = fileTask.getFile(); - - //clear return values from modules for last file - synchronized (abstractFileModulesRetValues) { - abstractFileModulesRetValues.clear(); - } - - //logger.log(Level.INFO, "IngestManager: Processing: {0}", fileToProcess.getName()); - - for (IngestModuleAbstractFile module : dataSourceTask.getModules()) { - //process the file with every file module - if (isCancelled()) { - logger.log(Level.INFO, "Terminating file ingest due to cancellation."); - return null; - } - progress.progress(fileToProcess.getName() + " (" + module.getName() + ")", processedFiles); - - try { - stats.logFileModuleStartProcess(module); - IngestModuleAbstractFile.ProcessResult result = module.process(filepipelineContext, fileToProcess); - stats.logFileModuleEndProcess(module); - - //store the result for subsequent modules for this file - synchronized (abstractFileModulesRetValues) { - abstractFileModulesRetValues.put(module.getName(), result); - } - - } catch (Exception e) { - logger.log(Level.SEVERE, "Error: unexpected exception from module: " + module.getName(), e); - stats.addError(module); - } catch (OutOfMemoryError e) { - logger.log(Level.SEVERE, "Error: out of memory from module: " + module.getName(), e); - stats.addError(module); - } - - } //end for every module - - //free the internal file resource after done with every module - fileToProcess.close(); - - // notify listeners thsi file is done - fireFileDone(fileToProcess.getId()); - - int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); - if (newTotalEnqueuedFiles > totalEnqueuedFiles) { - //update if new enqueued - totalEnqueuedFiles = newTotalEnqueuedFiles + 1;// + processedFiles + 1; - //processedFiles = 0; - //reset - progress.switchToIndeterminate(); - progress.switchToDeterminate(totalEnqueuedFiles); - } - if (processedFiles < totalEnqueuedFiles) { //fix for now to handle the same datasource Content enqueued twice - ++processedFiles; - } - //--totalEnqueuedFiles; - - - } //end of for every AbstractFile - logger.log(Level.INFO, "IngestManager: Finished processing files"); - return null; - } - - @Override - protected void done() { - try { - super.get(); //block and get all exceptions thrown while doInBackground() - //notify modules of completion - if (!this.isCancelled()) { - for (IngestModuleAbstractFile s : abstractFileModules) { - try { - s.complete(); - IngestManager.fireModuleEvent(IngestModuleEvent.COMPLETED.toString(), s.getName()); - } - catch (Exception ex) { - logger.log(Level.SEVERE, "Module " + s.getName() + " threw exception during call to complete()", ex); - } - } - } - - logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); - logger.log(Level.INFO, "Freeing jvm heap resources post file pipeline run"); - System.gc(); - logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); - - } catch (CancellationException e) { - //task was cancelled - handleInterruption(); - } catch (InterruptedException ex) { - handleInterruption(); - } catch (ExecutionException ex) { - handleInterruption(); - logger.log(Level.SEVERE, "Fatal error during ingest.", ex); - } catch (Exception ex) { - handleInterruption(); - logger.log(Level.SEVERE, "Fatal error during ingest.", ex); - } finally { - stats.end(); - progress.finish(); - - if (!this.isCancelled()) { - logger.log(Level.INFO, "Summary Report: " + stats.toString()); - logger.log(Level.INFO, "File module timings: " + stats.getFileModuleStats()); - if (ui != null) { - logger.log(Level.INFO, "Ingest messages count: " + ui.getMessagesCount()); - } - - IngestManager.this.postMessage(IngestMessage.createManagerMessage("File Ingest Complete", - stats.toHtmlString())); - } - } - - } - - private void handleInterruption() { - for (IngestModuleAbstractFile s : abstractFileModules) { - if (isModuleRunning(s)) { - try { - s.stop(); - } catch (Exception e) { - logger.log(Level.WARNING, "Exception while stopping module: " + s.getName(), e); - } - } - IngestManager.fireModuleEvent(IngestModuleEvent.STOPPED.toString(), s.getName()); - } - //empty queues - scheduler.getFileScheduler().empty(); - } - } - - /** - * Thread that adds content/file and module pairs to queues. - * Starts pipelines when done. */ - private class EnqueueWorker extends SwingWorker { - - private List modules; - private final List inputs; - private final Logger logger = Logger.getLogger(EnqueueWorker.class.getName()); - - EnqueueWorker(final List modules, final List inputs) { - this.modules = modules; - this.inputs = inputs; - } - private ProgressHandle progress; - - @Override - protected Object doInBackground() throws Exception { - - final String displayName = NbBundle - .getMessage(this.getClass(), "IngestManager.EnqueueWorker.displayName.text"); + // Set up a progress bar that can be used to cancel all of the + // ingest jobs currently being performed. + final String displayName = "Queueing ingest tasks"; progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { logger.log(Level.INFO, "Queueing ingest cancelled by user."); if (progress != null) { - progress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestManager.EnqueueWorker.process.cancelling", - displayName)); + progress.setDisplayName(displayName + " (Cancelling...)"); } - return EnqueueWorker.this.cancel(true); + IngestManager.getDefault().stopAll(); + return true; } }); - progress.start(2 * inputs.size()); - //progress.switchToIndeterminate(); - queueAll(modules, inputs); + progress.start(2 * dataSources.size()); + int processed = 0; + for (Content dataSource : dataSources) { + if (isCancelled()) { + logger.log(Level.INFO, "Task scheduling thread cancelled"); + return null; + } + + final String inputName = dataSource.getName(); + IngestJob ingestJob = new IngestJob(IngestManager.this.getNextDataSourceTaskId(), dataSource, moduleTemplates, processUnallocatedSpace); + + List errors = ingestJob.startUpIngestPipelines(); + if (!errors.isEmpty()) { + StringBuilder failedModules = new StringBuilder(); + for (int i = 0; i < errors.size(); ++i) { + IngestModuleError error = errors.get(i); + String moduleName = error.getModuleDisplayName(); + logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); + failedModules.append(moduleName); + if ((errors.size() > 1) && (i != (errors.size() - 1))) { + failedModules.append(","); + } + } + MessageNotifyUtil.Message.error( + "Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n" + + "No ingest modules will be run. Please disable the module " + + "or fix the error and restart ingest by right clicking on " + + "the data source and selecting Run Ingest Modules.\n\n" + + "Error: " + errors.get(0).getModuleError().getMessage()); + return null; + } + + // Save the ingest job for later cleanup of pipelines. + ingestJobs.put(ingestJob.getId(), ingestJob); + + // Queue the data source ingest tasks for the ingest job. + progress.progress("DataSource Ingest" + " " + inputName, processed); + scheduler.getDataSourceScheduler().schedule(ingestJob); + progress.progress("DataSource Ingest" + " " + inputName, ++processed); + + // Queue the file ingest tasks for the ingest job. + progress.progress("File Ingest" + " " + inputName, processed); + scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob); + progress.progress("File Ingest" + " " + inputName, ++processed); + } + return null; } - /* clean up or start the worker threads */ @Override protected void done() { try { - super.get(); //block and get all exceptions thrown while doInBackground() - } catch (CancellationException e) { - //task was cancelled - handleInterruption(e); - } catch (InterruptedException ex) { - handleInterruption(ex); - } catch (ExecutionException ex) { - handleInterruption(ex); - - + super.get(); + } catch (CancellationException | InterruptedException ex) { + // IngestManager.stopAll() will dispose of all tasks. } catch (Exception ex) { - handleInterruption(ex); - + logger.log(Level.SEVERE, "Error while scheduling ingest jobs", ex); + MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial"); } finally { - //queing end - if (this.isCancelled()) { - //empty queues - handleInterruption(new Exception()); - } else { - //start ingest workers + if (!isCancelled()) { startAll(); } progress.finish(); } } + } - /** - * Create modules and schedule analysis - * @param modules Modules to load into pipeline - * @param inputs List of parent data sources - */ - private void queueAll(List modules, final List inputs) { - - int processed = 0; - for (Content input : inputs) { - final String inputName = input.getName(); + /** + * Performs data source ingest tasks for one or more ingest jobs on a worker + * thread. + */ + class DataSourceTaskWorker extends SwingWorker { - // Create a new instance of the modules for each data source - final List dataSourceMods = new ArrayList(); - final List fileMods = new ArrayList(); + private final long id; + private ProgressHandle progress; - for (IngestModuleAbstract module : modules) { - if (isCancelled()) { - logger.log(Level.INFO, "Terminating ingest queueing due to cancellation."); - return; - } - - final String moduleName = module.getName(); - progress.progress(moduleName + " " + inputName, processed); - - switch (module.getType()) { - case DataSource: - final IngestModuleDataSource newModuleInstance = - (IngestModuleDataSource) moduleLoader.getNewIngestModuleInstance(module); - if (newModuleInstance != null) { - dataSourceMods.add(newModuleInstance); - } else { - logger.log(Level.INFO, "Error loading module and adding input " + inputName - + " with module " + module.getName()); - } - break; - - case AbstractFile: - //enqueue the same singleton AbstractFile module - logger.log(Level.INFO, "Adding input " + inputName - + " for AbstractFileModule " + module.getName()); - - fileMods.add((IngestModuleAbstractFile) module); - break; - - default: - logger.log(Level.SEVERE, "Unexpected module type: " + module.getType().name()); - } - - }//for modules - - - /* Schedule the data source-level ingest modules for this data source */ - final DataSourceTask dataSourceTask = - new DataSourceTask(input, dataSourceMods, getProcessUnallocSpace()); - - - logger.log(Level.INFO, "Queing data source ingest task: " + dataSourceTask); - progress.progress(NbBundle.getMessage(this.getClass(), "IngestManager.datatSourceIngest.progress.text", - inputName), processed); - final IngestScheduler.DataSourceScheduler dataSourceScheduler = scheduler.getDataSourceScheduler(); - dataSourceScheduler.schedule(dataSourceTask); - progress.progress(NbBundle.getMessage(this.getClass(), "IngestManager.datatSourceIngest.progress.text", - inputName), ++processed); - - - /* Schedule the file-level ingest modules for the children of the data source */ - final DataSourceTask fTask = - new DataSourceTask(input, fileMods, getProcessUnallocSpace()); - - logger.log(Level.INFO, "Queing file ingest task: " + fTask); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.fileIngest.progress.text", inputName), processed); - final IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler(); - fileScheduler.schedule(fTask); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.fileIngest.progress.text", inputName), ++processed); - - } //for data sources + DataSourceTaskWorker(long threadId) { + this.id = threadId; } - private void handleInterruption(Exception ex) { - logger.log(Level.SEVERE, "Error while enqueing files. ", ex); - //empty queues - scheduler.getFileScheduler().empty(); - scheduler.getDataSourceScheduler().empty(); + @Override + protected Void doInBackground() throws Exception { + logger.log(Level.INFO, "Data source ingest thread (id={0}) started", this.id); + + // Set up a progress bar that can be used to cancel all of the + // ingest jobs currently being performed. + progress = ProgressHandleFactory.createHandle("Data source ingest", new Cancellable() { + @Override + public boolean cancel() { + logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", DataSourceTaskWorker.this.id); + if (progress != null) { + progress.setDisplayName(NbBundle.getMessage(this.getClass(), + "IngestManager.DataSourceTaskWorker.process.cancelling", + "Data source ingest")); + } + IngestManager.getDefault().stopAll(); + return true; + } + }); + progress.start(); + progress.switchToIndeterminate(); + + IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler(); + while (scheduler.hasNext()) { + if (isCancelled()) { + logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id); + return null; + } + + IngestJob ingestJob = scheduler.next(); + IngestJob.DataSourceIngestPipeline pipeline = ingestJob.getDataSourceIngestPipelineForThread(this.id); + pipeline.process(this, this.progress); + } + + logger.log(Level.INFO, "Data source ingest thread (id={0}) completed", this.id); + IngestManager.getDefault().reportThreadDone(this.id); + return null; + } + + @Override + protected void done() { + try { + super.get(); + } catch (CancellationException | InterruptedException e) { + logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id); + IngestManager.getDefault().reportThreadDone(this.id); + } catch (Exception ex) { + String message = String.format("Data source ingest thread (id=%d) experienced a fatal error", this.id); + logger.log(Level.SEVERE, message, ex); + IngestManager.getDefault().reportThreadDone(this.id); + } finally { + progress.finish(); + } + } + } + + /** + * Performs file ingest tasks for one or more ingest jobs on a worker + * thread. + */ + class FileTaskWorker extends SwingWorker { + + private final long id; + private ProgressHandle progress; + + FileTaskWorker(long threadId) { + this.id = threadId; + } + + @Override + protected Object doInBackground() throws Exception { + logger.log(Level.INFO, "File ingest thread (id={0}) started", this.id); + + // Set up a progress bar that can be used to cancel all of the + // ingest jobs currently being performed. + final String displayName = NbBundle + .getMessage(this.getClass(), "IngestManager.FileTaskWorker.displayName"); + progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", FileTaskWorker.this.id); + if (progress != null) { + progress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestManager.FileTaskWorker.process.cancelling", + displayName)); + } + IngestManager.getDefault().stopAll(); + return true; + } + }); + progress.start(); + progress.switchToIndeterminate(); + IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); + int totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); + progress.switchToDeterminate(totalEnqueuedFiles); + + int processedFiles = 0; + while (fileScheduler.hasNext()) { + if (isCancelled()) { + IngestManager.getDefault().reportThreadDone(this.id); + logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id); + return null; + } + + IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); + AbstractFile file = task.getFile(); + progress.progress(file.getName(), processedFiles); + IngestJob.FileIngestPipeline pipeline = task.getJob().getFileIngestPipelineForThread(this.id); + pipeline.process(file); + + // Update the progress bar. + int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); + if (newTotalEnqueuedFiles > totalEnqueuedFiles) { + totalEnqueuedFiles = newTotalEnqueuedFiles + 1; + progress.switchToIndeterminate(); + progress.switchToDeterminate(totalEnqueuedFiles); + } + if (processedFiles < totalEnqueuedFiles) { + ++processedFiles; + } + } + + logger.log(Level.INFO, "File ingest thread (id={0}) completed", this.id); + IngestManager.getDefault().reportThreadDone(this.id); + return null; + } + + @Override + protected void done() { + try { + super.get(); + } catch (CancellationException | InterruptedException e) { + logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id); + IngestManager.getDefault().reportThreadDone(this.id); + } catch (Exception ex) { + String message = String.format("File ingest thread {0} experienced a fatal error", this.id); + logger.log(Level.SEVERE, message, ex); + IngestManager.getDefault().reportThreadDone(this.id); + } finally { + progress.finish(); + } } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessage.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessage.java index 77e22204e1..eb695fcc7f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessage.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessage.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.ingest; import java.text.SimpleDateFormat; import java.util.Date; - import org.openide.util.NbBundle; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -41,7 +40,7 @@ public class IngestMessage { private long ID; private MessageType messageType; - private IngestModuleAbstract source; + private String source; private String subject; private String detailsHtml; private String uniqueKey; @@ -53,7 +52,7 @@ public class IngestMessage { /** * Private constructor used by factory methods */ - private IngestMessage(long ID, MessageType messageType, IngestModuleAbstract source, String subject, String detailsHtml, String uniqueKey) { + private IngestMessage(long ID, MessageType messageType, String source, String subject, String detailsHtml, String uniqueKey) { this.ID = ID; this.source = source; this.messageType = messageType; @@ -70,7 +69,7 @@ public class IngestMessage { return ID; } - public IngestModuleAbstract getSource() { + public String getSource() { return source; } @@ -105,7 +104,7 @@ public class IngestMessage { sb.append(NbBundle.getMessage(this.getClass(), "IngestMessage.toString.type.text", messageType.name())); if (source != null) //can be null for manager messages { - sb.append(NbBundle.getMessage(this.getClass(), "IngestMessage.toString.source.text", source.getName())); + sb.append(source); } sb.append( NbBundle.getMessage(this.getClass(), "IngestMessage.toString.date.text", dateFormat.format(datePosted))); @@ -176,7 +175,7 @@ public class IngestMessage { * @param detailsHtml html formatted detailed message (without leading and closing <html> tags), for instance, a human-readable representation of the data. Or null. * @return */ - public static IngestMessage createMessage(long ID, MessageType messageType, IngestModuleAbstract source, String subject, String detailsHtml) { + public static IngestMessage createMessage(long ID, MessageType messageType, String source, String subject, String detailsHtml) { if (messageType == null || source == null || subject == null) { throw new IllegalArgumentException( NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.typeSrcSubjNotNull.msg")); @@ -192,7 +191,7 @@ public class IngestMessage { * @param subject message subject to be displayed * @return */ - public static IngestMessage createMessage(long ID, MessageType messageType, IngestModuleAbstract source, String subject) { + public static IngestMessage createMessage(long ID, MessageType messageType, String source, String subject) { return createMessage(ID, messageType, source, subject, null); } @@ -205,7 +204,7 @@ public class IngestMessage { * @param detailsHtml html formatted detailed message (without leading and closing <html> tags), for instance, a human-readable representation of the data. Or null * @return */ - public static IngestMessage createErrorMessage(long ID, IngestModuleAbstract source, String subject, String detailsHtml) { + public static IngestMessage createErrorMessage(long ID, String source, String subject, String detailsHtml) { if (source == null || subject == null) { throw new IllegalArgumentException( NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.srcSubjNotNull.msg")); @@ -221,7 +220,7 @@ public class IngestMessage { * @param detailsHtml html formatted detailed message (without leading and closing <html> tags), for instance, a human-readable representation of the data. Or null * @return */ - public static IngestMessage createWarningMessage(long ID, IngestModuleAbstract source, String subject, String detailsHtml) { + public static IngestMessage createWarningMessage(long ID, String source, String subject, String detailsHtml) { if (source == null || subject == null) { throw new IllegalArgumentException( NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.srcSubjNotNull.msg")); @@ -239,7 +238,7 @@ public class IngestMessage { * @param data blackboard artifact associated with the message, the same as fired in ModuleDataEvent by the module * @return */ - public static IngestMessage createDataMessage(long ID, IngestModuleAbstract source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data) { + public static IngestMessage createDataMessage(long ID, String source, String subject, String detailsHtml, String uniqueKey, BlackboardArtifact data) { if (source == null || subject == null || detailsHtml == null || data == null) { throw new IllegalArgumentException( NbBundle.getMessage(IngestMessage.class, "IngestMessage.exception.srcSubjDetailsDataNotNull.msg")); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java index 40e409c81c..7596e1f4cd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,10 +35,8 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; import javax.swing.JLabel; import javax.swing.JPanel; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.JTable; @@ -54,6 +52,7 @@ import javax.swing.table.TableCellRenderer; import org.sleuthkit.autopsy.ingest.IngestMessage.*; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import java.util.logging.Level; /** * Notification window showing messages from modules to user @@ -260,7 +259,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { //this should be done at the end to make it easy to initialize before events are handled tableModel.addTableModelListener(this); - } @Override @@ -285,15 +283,12 @@ class IngestMessagePanel extends JPanel implements TableModelListener { } } - public synchronized void addMessage(IngestMessage m) { - //final int origMsgUnreadUnique = tableModel.getNumberUnreadGroups(); tableModel.addMessage(m); //update total individual messages count ++totalMessages; final int newMsgUnreadUnique = tableModel.getNumberUnreadGroups(); - try { messagePcs.firePropertyChange(TOTAL_NUM_MESSAGES_CHANGED, 0, newMsgUnreadUnique); @@ -310,10 +305,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { this.totalMessagesNameVal.setText(Long.toString(totalMessages)); final int totalMessagesUnique = tableModel.getNumberGroups(); this.totalUniqueMessagesNameVal.setText(Integer.toString(totalMessagesUnique)); - //this.unreadLabelVal.setText(Integer.toString(newMsgUnreadUnique)); - - //autoscroll - //messageTable.scrollRectToVisible(messageTable.getCellRect(messageTable.getRowCount() - 1, messageTable.getColumnCount(), true)); } public synchronized void clearMessages() { @@ -342,7 +333,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { private synchronized void setVisited(int rowNumber) { final int origMsgGroups = tableModel.getNumberUnreadGroups(); tableModel.setVisited(rowNumber); - //renderer.setSelected(rowNumber); lastRowSelected = rowNumber; try { @@ -381,9 +371,9 @@ class IngestMessagePanel extends JPanel implements TableModelListener { NbBundle.getMessage(this.getClass(), "IngestMessagePanel.MsgTableMod.colNames.new"), NbBundle.getMessage(this.getClass(), "IngestMessagePanel.MsgTableMod.colNames.subject"), NbBundle.getMessage(this.getClass(), "IngestMessagePanel.MsgTableMod.colNames.timestamp")}; - private List messageData = new ArrayList(); + private List messageData = new ArrayList<>(); //for keeping track of messages to group, per module, by uniqness - private Map>> groupings = new HashMap>>(); + private Map>> groupings = new HashMap<>(); private boolean chronoSort = true; //chronological sort default private static final int MESSAGE_GROUP_THRESH = 3; //group messages after 3 messages per module with same uniqness private Logger logger = Logger.getLogger(MessageTableModel.class.getName()); @@ -393,18 +383,11 @@ class IngestMessagePanel extends JPanel implements TableModelListener { } private void init() { - final IngestManager manager = IngestManager.getDefault(); - //initialize groupings map with modules - for (IngestModuleAbstract module : manager.enumerateAbstractFileModules()) { - groupings.put(module, new HashMap>()); - } - for (IngestModuleAbstract module : manager.enumerateDataSourceModules()) { - groupings.put(module, new HashMap>()); + List moduleFactories = IngestModuleFactoryLoader.getInstance().getIngestModuleFactories(); + for (IngestModuleFactory factory : moduleFactories) { + groupings.put(factory.getModuleDisplayName(), new HashMap>()); } } - - - @Override public int getColumnCount() { @@ -486,8 +469,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { switch (columnIndex) { case 0: - Object module = entry.messageGroup.getSource(); - ret = module == null ? "" : entry.messageGroup.getSource().getName(); + ret = entry.messageGroup.getSource(); break; case 1: ret = entry.messageGroup.getCount(); @@ -502,7 +484,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { ret = entry.messageGroup.getDatePosted(); break; default: - logger.log(Level.SEVERE, "Invalid table column index: " + columnIndex); + logger.log(Level.SEVERE, "Invalid table column index: {0}", columnIndex); break; } return ret; @@ -534,17 +516,17 @@ class IngestMessagePanel extends JPanel implements TableModelListener { synchronized public void addMessage(IngestMessage m) { //check how many messages per module with the same uniqness //and add to existing group or create a new group - IngestModuleAbstract module = m.getSource(); + String moduleName = m.getSource(); IngestMessageGroup messageGroup = null; - if (module != null && m.getMessageType() == IngestMessage.MessageType.DATA) { + if (moduleName != null && m.getMessageType() == IngestMessage.MessageType.DATA) { //not a manager message, a data message, then group - final Map> groups = groupings.get(module); + final Map> groups = groupings.get(moduleName); //groups for this uniqueness final String uniqueness = m.getUniqueKey(); List uniqGroups = groups.get(uniqueness); if (uniqGroups == null) { //first one with this uniqueness - uniqGroups = new ArrayList(); + uniqGroups = new ArrayList<>(); messageGroup = new IngestMessageGroup(m); uniqGroups.add(messageGroup); groups.put(uniqueness, uniqGroups); @@ -565,7 +547,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { uniqGroups.add(messageGroup); //remove all rows with this uniquness, new merged row will be added to the bottom - int toRemove = 0; + int toRemove; while ((toRemove = getTableEntryIndex(uniqueness)) != -1) { messageData.remove(toRemove); //remove the row, will be added to the bottom @@ -715,7 +697,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { private List messages; IngestMessageGroup(IngestMessage message) { - messages = new ArrayList(); + messages = new ArrayList<>(); messages.add(message); } @@ -768,7 +750,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { } else { return LOW_PRI_COLOR; } - } /** @@ -798,7 +779,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener { /* * return source module, should be the same for all msgs */ - IngestModuleAbstract getSource() { + String getSource() { return messages.get(0).getSource(); } @@ -889,7 +870,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { return cell; } - } /** @@ -965,7 +945,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { return cell; } - } /** @@ -973,8 +952,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { */ private class MessageVisitedSelection implements ListSelectionListener { - private Logger logger = Logger.getLogger(MessageVisitedSelection.class.getName()); - @Override public void valueChanged(ListSelectionEvent e) { ListSelectionModel selModel = (ListSelectionModel) e.getSource(); @@ -1003,7 +980,6 @@ class IngestMessagePanel extends JPanel implements TableModelListener { } messageTable.setCursor(null); } - } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java new file mode 100755 index 0000000000..970677ec9e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -0,0 +1,47 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +/** + * The interface that must be implemented by all ingest modules. + */ +public interface IngestModule { + + public enum ProcessResult { + OK, + ERROR + }; + + public class IngestModuleException extends Exception { + public IngestModuleException(String message) { + super(message); + } + } + + // RJCTODO: Write header comment, make sure to mention "one module instance per thread" + /** + * Invoked by the ingest frame + * @param context + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + */ + void startUp(IngestJobContext context) throws IngestModuleException; + + // RJCTODO: Write header comment, make sure to mention "one module instance per thread" + void shutDown(boolean ingestJobWasCancelled); +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAbstract.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAbstract.java deleted file mode 100644 index 8ee8baea01..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAbstract.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011-2012 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.sleuthkit.autopsy.ingest; - - -/** - * Base interface for ingest modules - */ - public abstract class IngestModuleAbstract { - - private String args; - - /** - * Possible module types for the implementing classes - */ - public enum ModuleType { - /** - * DataSource type module - */ - DataSource, - - /** - * AbstractFile type module - */ - AbstractFile - }; - - public class IngestModuleException extends Exception { - public IngestModuleException(String msg) { - super(msg); - } - } - - - /** - * Invoked every time an ingest session is started by the framework. - * A module should support multiple invocations of init() throughout the application life-cycle. - * In this method, the module should reinitialize its internal objects and resources and get them ready - * for a brand new ingest processing. - * - * Here are some things you may do in this method if you'll need them later. - * - Get a handle to the ingest services using org.sleuthkit.autopsy.ingest.IngestServices.getDefault(). - * - Get the current case using org.sleuthkit.autopsy.ingest.IngestServices.getCurrentSleuthkitCaseDb(). - * - * NEVER initialize IngestServices handle in the member declaration, because it might result - * in multiple instances of the singleton -- different class loaders are used in different modules. - * @param initContext context used to initialize some modules - * - * @throws IngestModuleException if a critical error occurs in initializing the module. - */ - abstract public void init(IngestModuleInit initContext) throws IngestModuleException; - - /** - * Invoked when an ingest session completes. - * The module should perform any resource (files, handles, caches) - * cleanup in this method and submit final results and post a final ingest inbox message. - */ - abstract public void complete(); - - /** - * Invoked on a module when an ingest session is interrupted by the user or system. - * The method implementation should be similar to complete() in that the - * module should perform any cleanup work. - * If there is pending data to be processed or pending results to be reported by the module - * then the results should be rejected and ignored and the method should return as early as possible. - * It should ensure it is in a defined state so that ingest can be rerun later. - */ - abstract public void stop(); - - /** - * Returns unique name of the module. Should not have collisions. - * @return unique module name - */ - abstract public String getName(); - - /** - * Gets the module version - * @return module version string - */ - abstract public String getVersion(); - - /** - * Gets user-friendly description of the module - * @return module description - */ - abstract public String getDescription(); - - /** - * Returns type of the module (data source-level or file-level) - * @return module type - */ - abstract public ModuleType getType(); - - - /** - * A module can manage and use additional threads to perform some work in the background. - * This method provides insight to the manager if the module has truly completed its work or not. - * - * - * @return true if any background threads/workers managed by this module are still running or are pending to be run, - * false if all work has been done, or if background threads are not used/managed by this module - */ - abstract public boolean hasBackgroundJobsRunning(); - - - /** - * Used to determine if a module has implemented a simple (run-time) - * configuration panel that is displayed by the ingest manager. - * - * @return true if this module has a simple (run-time) configuration - */ - public boolean hasSimpleConfiguration() { - return false; - } - - /** - * Used to determine if a module has implemented an advanced (general) - * configuration that can be used for more in-depth module configuration. - * - * @return true if this module has an advanced configuration - */ - public boolean hasAdvancedConfiguration() { - return false; - } - - /** - * Called by the ingest manager if the simple (run-time) configuration - * panel should save its current state so that the settings can be used - * during the ingest. - */ - public void saveSimpleConfiguration() {} - - /** - * If module implements advanced configuration panel - * it should read its current state and make it persistent / save it in this method - * so that the new configuration will be in effect during the ingest. - */ - public void saveAdvancedConfiguration() {} - - - /** - * Returns a panel that displays the simple (run-time) configuration for the - * given configuration context (such as pipeline instance). This is - * presented to the user before ingest starts and only basic settings should - * be given here. Use the advanced (general) configuration panel for more - * in-depth interfaces. The module (or its configuration controller object) - * is responsible for preserving / saving its configuration state In - * addition, saveSimpleConfiguration() can be used as the trigger. - * - * @param context the configuration context to use in the panel - * @return JPanel containing basic configuration widgets or null if simple - * configuration is not available - */ - public javax.swing.JPanel getSimpleConfiguration(String context) { - return null; - } - - /** - * Returns a panel that displays the advanced (run-time) configuration for - * the given configuration context (such as pipeline instance). Implements - * advanced module configuration exposed to the user before ingest starts. - * - * The module (or its configuration controller object) is responsible for - * preserving / saving its configuration state In addition, - * saveAdvancedConfiguration() can be used as the trigger. - * - * @param context the configuration context to use in the panel - * @return JPanel containing advanced configuration widgets or null if - * advanced configuration is not available - */ - public javax.swing.JPanel getAdvancedConfiguration(String context) { - return null; - } - } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAbstractFile.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAbstractFile.java deleted file mode 100644 index aa836763f6..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAbstractFile.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import org.sleuthkit.datamodel.AbstractFile; - -/** - * Ingest module interface that will be called for every file in the data source Content - */ -public abstract class IngestModuleAbstractFile extends IngestModuleAbstract { - - /** - * Return value resulting from processing AbstractFile - * If ERROR, can be used subsequent module - * in the pipeline as a hint to stop processing the file - */ - public enum ProcessResult { - OK, ///< Indicates that processing was successful (including if the file was largely ignored by the module) - ERROR, ///< Indicates that an error was encountered while processing the file, hint for later modules that depend on this module to skip processing the file due to error condition (such as file could not be read) - UNKNOWN ///< Indicates that a return value for the module is not known. This should not be returned directly by modules, but is used to indicate the module has not set its return value (e.g. it never ran) - }; - - @Override - public ModuleType getType() { - return ModuleType.AbstractFile; - } - - /** - * Entry point to process file / directory by the module. - * - * @param pipelineContext the context in which the ingest runs (with its own settings, modules, etc) - * @param abstractFile file to process - * @return ProcessResult result of the processing that can be used in the pipeline as a hint whether to further process this file - */ - abstract public ProcessResult process(PipelineContextpipelineContext, AbstractFile abstractFile); -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java new file mode 100755 index 0000000000..93379de7d4 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java @@ -0,0 +1,34 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +/** + * An adapter that provides a default implementation of the IngestModule + * interface. + */ +public abstract class IngestModuleAdapter implements IngestModule { + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + } + + @Override + public void shutDown(boolean ingestJobCancelled) { + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleDataSource.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleDataSource.java deleted file mode 100644 index 5ffdbc00b5..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleDataSource.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2011 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import org.sleuthkit.datamodel.Content; - -/** - * Ingest module that acts on entire image or set of logical files. - * These modules are for analysis tasks that do not operate on all files in the disk image or set of logical files. - * A new instance of this module will be created for each data source that is added. - * Therefore, data source-level modules can assume that the process() method will be called at most once after init() is called. - */ -public abstract class IngestModuleDataSource extends IngestModuleAbstract { - - @Override - public ModuleType getType() { - return ModuleType.DataSource; - } - - /** - * Called with the data source Content object to analyze. - * - * Modules typically use FileManager to get specific files to analyze. - * - * Results should be posted to the blackboard. - * The module should also send messages to the ingest inbox with interesting events (data, errors, warnings, infos). - * The module notifies data viewers by firing events using IngestManagerProxy.fireModuleDataEvent - * - * The module will have its own progress bar while it is running and it should update it with the IngestDataSourceWorkerController object. - * - * @param pipelineContext Context in which the ingest pipeline is running (Settings, modules, etc) - * @param dataSource data source to process (such as Image, VirtualDirectory for logical files etc, etc) - * @param controller Used to update progress bar and to check if the task has been canceled. - */ - abstract public void process(PipelineContextpipelineContext, Content dataSource, IngestDataSourceWorkerController controller); -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java new file mode 100755 index 0000000000..9c9910b405 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java @@ -0,0 +1,41 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +/** + * Encapsulates an exception thrown by an ingest module during an operation such + * as startup or shut down with an exception object for the error that occurred. + */ +final class IngestModuleError { + private final String moduleDisplayName; + private final Exception error; + + IngestModuleError(String moduleDisplayName, Exception error) { + this.moduleDisplayName = moduleDisplayName; + this.error = error; + } + + String getModuleDisplayName() { + return this.moduleDisplayName; + } + + Exception getModuleError() { + return this.error; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java new file mode 100755 index 0000000000..7070309021 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java @@ -0,0 +1,216 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +/** + * An interface that must be implemented by all providers of ingest modules. An + * ingest module factory will be used to create instances of a type of data + * source ingest module, a type of file ingest module, or both. + *

+ * IMPORTANT: The factory should be stateless to support context-sensitive use + * of the factory. The ingest framework is responsible for managing context + * switching and the persistence of resource configurations and per ingest job + * options. + *

+ * IMPORTANT: The ingest framework will create one or more instances of each + * module type for each ingest job it performs. The ingest framework may use + * multiple threads to complete an ingest job, but it is guaranteed that there + * will be no more than one module instance per thread. However, if these + * instances must share resources, the modules are responsible for synchronizing + * access to the shared resources and doing reference counting as required to + * release those resources correctly. + *

+ * IMPORTANT: To be discovered at runtime by the ingest framework, + * IngestModuleFactory implementations must be marked with the following + * NetBeans Service provider annotation: + * + * @ServiceProvider(service=IngestModuleFactory.class) + */ +public interface IngestModuleFactory { + + /** + * Gets the display name that identifies the family of ingest modules the + * factory creates. + * + * @return The module family display name. + */ + String getModuleDisplayName(); + + /** + * Gets a brief, user-friendly description of the family of ingest modules + * the factory creates. + * + * @return The module family description. + */ + String getModuleDescription(); + + /** + * Gets the version number of the family of ingest modules the factory + * creates. + * + * @return The module family version number. + */ + String getModuleVersionNumber(); + + /** + * Queries the factory to determine if it provides user interface panels to + * configure resources to be used by instances of the family of ingest + * modules the factory creates. For example, the core hash lookup ingest + * module factory provides resource configuration panels to import and + * create hash databases. The hash databases are then enabled or disabled + * per ingest job using ingest job options panels. If the module family does + * not have a resources configuration, the factory should extend + * IngestModuleFactoryAdapter to get an implementation of this method that + * returns false. + * + * @return True if the factory provides resource configuration panels. + */ + boolean hasGlobalSettingsPanel(); + + /** + * Gets a user interface panel that can be used to configure resources for + * instances of the family of ingest modules the factory creates. For + * example, the core hash lookup ingest module factory provides a resource + * configuration panel to import and create hash databases. The imported + * hash databases are then enabled or disabled per ingest job using ingest + * options panels. If the module family does not have a resources + * configuration, the factory should extend IngestModuleFactoryAdapter to + * get an implementation of this method that throws an + * UnsupportedOperationException. + *

+ * IMPORTANT: The ingest framework assumes that ingest module factories are + * stateless to support context-sensitive use of the factory, with the + * ingest framework managing context switching and the persistence of + * resource configurations and per ingest job options. A factory should not + * retain references to the resources configuration panels it creates. + * + * @param resourcesConfig A resources configuration with which to initialize + * the panel. + * @return A user interface panel for configuring ingest module resources. + */ + IngestModuleGlobalSetttingsPanel getGlobalSettingsPanel(); + + /** + * Gets the default per ingest job options for instances of the family of + * ingest modules the factory creates. For example, the core hash lookup + * ingest modules family has a resources configuration consisting of hash + * databases, all of which are enabled by default for an ingest job. If the + * module family does not have per ingest job options, the factory should + * extend IngestModuleFactoryAdapter to get an implementation of this method + * that returns an instance of the NoIngestJobOptions class. + * + * @return The ingest options. + */ + IngestModuleSettings getDefaultModuleSettings(); + + /** + * Queries the factory to determine if it provides user interface panels to + * set per ingest job options for instances of the family of ingest modules + * the factory creates. For example, the core hash lookup ingest module + * factory provides ingest options panels to enable or disable hash + * databases per ingest job. If the module family does not have per ingest + * job options, the factory should extend IngestModuleFactoryAdapter to get + * an implementation of this method that returns false. + * + * @return True if the factory provides ingest job options panels. + */ + boolean hasModuleSettingsPanel(); + + /** + * Gets a user interface panel that can be used to set per ingest job + * options for instances of the family of ingest modules the factory + * creates. For example, the core hash lookup ingest module factory provides + * ingest options panels to enable or disable hash databases per ingest job. + * If the module family does not have ingest job options, the factory should + * extend IngestModuleFactoryAdapter to get an implementation of this method + * that throws an UnsupportedOperationException. + *

+ * IMPORTANT: The ingest framework assumes that ingest module factories are + * stateless to support context-sensitive use of the factory. The ingest + * framework is responsible for managing context switching and the + * persistence of resource configurations and per ingest job options. A + * factory should not retain references to the ingest job options panels it + * creates. + * + * @param resourcesConfig + * @param ingestOptions Per ingest job options to initialize the panel. + * @return A user interface panel. + */ + IngestModuleSettingsPanel getModuleSettingsPanel(IngestModuleSettings settings); + + /** + * Queries the factory to determine if it is capable of creating file ingest + * modules. + * + * @return True if the factory can create file ingest modules. + */ + boolean isDataSourceIngestModuleFactory(); + + /** + * Creates a data source ingest module instance. + *

+ * IMPORTANT: The factory should be stateless to support context-sensitive + * use of the factory. The ingest framework is responsible for managing + * context switching and the persistence of resource configurations and per + * ingest job options. A factory should not retain references to the data + * source ingest module instances it creates. + *

+ * IMPORTANT: The ingest framework will create one or more data source + * ingest module instances for each ingest job it performs. The ingest + * framework may use multiple threads to complete an ingest job, but it is + * guaranteed that there will be no more than one module instance per + * thread. However, if these instances must share resources, the modules are + * responsible for synchronizing access to the shared resources and doing + * reference counting as required to release those resources correctly. + * + * @param ingestOptions The ingest options for the module instance. + * @return A data source ingest module instance. + */ + DataSourceIngestModule createDataSourceIngestModule(IngestModuleSettings settings); + + /** + * Queries the factory to determine if it is capable of creating file ingest + * module instances. + * + * @return True if the factory can create file ingest module instances. + */ + boolean isFileIngestModuleFactory(); + + /** + * Creates a file ingest module instance. + *

+ * IMPORTANT: The factory should be stateless to support context-sensitive + * use of the factory. The ingest framework is responsible for managing + * context switching and the persistence of resource configurations and per + * ingest job options. A factory should not retain references to the file + * ingest module instances it creates. + *

+ * IMPORTANT: The ingest framework will create one or more file ingest + * module instances for each ingest job it performs. The ingest framework + * may use multiple threads to complete an ingest job, but it is guaranteed + * that there will be no more than one module instance per thread. However, + * if these instances must share resources, the modules are responsible for + * synchronizing access to the shared resources and doing reference counting + * as required to release those resources correctly. + * + * @param ingestOptions The ingest options for the module instance. + * @return A file ingest module instance. + */ + FileIngestModule createFileIngestModule(IngestModuleSettings settings); +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java new file mode 100755 index 0000000000..e750e89b45 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java @@ -0,0 +1,80 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +/** + * An adapter that provides no-op implementations of various IngestModuleFactory + * methods. + */ +public abstract class IngestModuleFactoryAdapter implements IngestModuleFactory { + + @Override + public abstract String getModuleDisplayName(); + + @Override + public abstract String getModuleDescription(); + + @Override + public abstract String getModuleVersionNumber(); + + @Override + public boolean hasGlobalSettingsPanel() { + return false; + } + + @Override + public IngestModuleGlobalSetttingsPanel getGlobalSettingsPanel() { + throw new UnsupportedOperationException(); + } + + @Override + public IngestModuleSettings getDefaultModuleSettings() { + return new NoIngestModuleSettings(); + } + + @Override + public boolean hasModuleSettingsPanel() { + return false; + } + + @Override + public IngestModuleSettingsPanel getModuleSettingsPanel(IngestModuleSettings ingestOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isDataSourceIngestModuleFactory() { + return false; + } + + @Override + public DataSourceIngestModule createDataSourceIngestModule(IngestModuleSettings ingestOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isFileIngestModuleFactory() { + return false; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleSettings ingestOptions) { + throw new UnsupportedOperationException(); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java new file mode 100644 index 0000000000..49813595c0 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java @@ -0,0 +1,62 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.logging.Level; +import org.openide.util.Lookup; +import org.sleuthkit.autopsy.coreutils.Logger; + +/** + * Looks up loaded ingest module factories using the NetBean global lookup. + */ +final class IngestModuleFactoryLoader { + + private static final Logger logger = Logger.getLogger(IngestModuleFactoryLoader.class.getName()); + private static IngestModuleFactoryLoader instance; + + private IngestModuleFactoryLoader() { + } + + synchronized static IngestModuleFactoryLoader getInstance() { + if (instance == null) { + instance = new IngestModuleFactoryLoader(); + } + return instance; + } + + synchronized List getIngestModuleFactories() { + List moduleFactories = new ArrayList<>(); + HashSet moduleDisplayNames = new HashSet<>(); + Collection factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class); + for (IngestModuleFactory factory : factories) { + logger.log(Level.INFO, "Found ingest module factory: name = {0}, version = {1}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()}); + if (!moduleDisplayNames.contains(factory.getModuleDisplayName())) { + moduleFactories.add(factory); + moduleDisplayNames.add(factory.getModuleDisplayName()); + } else { + logger.log(Level.SEVERE, "Found duplicate ingest module display name, discarding ingest module factory (name = {0}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()}); + } + } + return new ArrayList<>(moduleFactories); + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleGlobalSetttingsPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleGlobalSetttingsPanel.java new file mode 100755 index 0000000000..160ab6df83 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleGlobalSetttingsPanel.java @@ -0,0 +1,29 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import javax.swing.JPanel; + +/** + * Base class for ingest module resources configuration panels. + */ +public abstract class IngestModuleGlobalSetttingsPanel extends JPanel { + + public abstract void saveSettings(); +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java deleted file mode 100644 index d5ba207771..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java +++ /dev/null @@ -1,1121 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2012 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import java.beans.PropertyChangeListener; -import java.beans.PropertyChangeSupport; -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLDecoder; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import org.openide.modules.ModuleInfo; -import org.openide.util.*; -import org.reflections.Reflections; -import org.reflections.scanners.ResourcesScanner; -import org.reflections.scanners.SubTypesScanner; -import org.reflections.util.ConfigurationBuilder; -import org.reflections.util.FilterBuilder; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.ModuleSettings; -import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.coreutils.XMLUtil; -import org.w3c.dom.Comment; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -/** - * Class responsible for discovery and loading ingest modules specified in - * pipeline XML file. Maintains a singleton instance. Requires restart of - * application for changes in XML to take effect. - * - * Supports module auto-discovery from system-wide and user-dir wide jar files. - * Discovered modules are validated, and if valid, they are added to end of - * configuration and saved in the XML. - * - * If module is removed/uninstalled, it will remain in the XML file, but it will - * not load because it will fail the validation. - * - * Get a handle to the object by calling static getDefault() method. The - * singleton instance will initialize itself the first time - it will load XML - * and autodiscover currently present ingest modules in the jar classpath.. - * - * - * Refer to - * http://sleuthkit.org/sleuthkit/docs/framework-docs/pipeline_config_page.html - * for the pipeline XML fiel schema details. - * - * NOTE: this will be part of future IngestPipelineManager with IngestManager - * code refactored - */ - final class IngestModuleLoader { - - private static final String PIPELINE_CONFIG_XML = "pipeline_config.xml"; - private static final String XSDFILE = "PipelineConfigSchema.xsd"; - private String absFilePath; - private static IngestModuleLoader instance; - //raw XML pipeline representation for validation - private final List pipelinesXML; - //validated pipelines with instantiated modules - private final List filePipeline; - private final List dataSourcePipeline; - private static final Logger logger = Logger.getLogger(IngestModuleLoader.class.getName()); - private ClassLoader classLoader; - private PropertyChangeSupport pcs; - private static final String ENCODING = "UTF-8"; - private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; - private SimpleDateFormat dateFormatter; - //used to specify default unique module order of autodiscovered modules - //if not specified - private int numModDiscovered = 0; - private static String CUR_MODULES_DISCOVERED_SETTING = "curModulesDiscovered"; - - //events supported - enum Event { - - ModulesReloaded - }; - - private IngestModuleLoader() { - pipelinesXML = new ArrayList(); - filePipeline = new ArrayList(); - dataSourcePipeline = new ArrayList(); - dateFormatter = new SimpleDateFormat(DATE_FORMAT); - - String numModDiscoveredStr = ModuleSettings.getConfigSetting(IngestManager.MODULE_PROPERTIES, CUR_MODULES_DISCOVERED_SETTING); - if (numModDiscoveredStr != null) { - try { - numModDiscovered = Integer.valueOf(numModDiscoveredStr); - } catch (NumberFormatException e) { - numModDiscovered = 0; - logger.log(Level.WARNING, "Could not parse numModDiscovered setting, defaulting to 0", e); - } - } - - pcs = new PropertyChangeSupport(this); - registerModulesChange(); - } - - synchronized static IngestModuleLoader getDefault() throws IngestModuleLoaderException { - if (instance == null) { - logger.log(Level.INFO, "Creating ingest module loader instance"); - instance = new IngestModuleLoader(); - instance.init(); - } - return instance; - } - - /** - * Add a listener to listen for modules reloaded events such as when new - * modules have been added / removed / reconfigured - * - * @param l listener to add - */ - void addModulesReloadedListener(PropertyChangeListener l) { - pcs.addPropertyChangeListener(l); - } - - /** - * Remove a listener to listen for modules reloaded events such as when new - * modules have been added / removed / reconfigured - * - * @param l listener to remove - */ - void removeModulesReloadedListener(PropertyChangeListener l) { - pcs.removePropertyChangeListener(l); - } - - /** - * validate raw pipeline, set valid to true member on pipeline and modules - * if valid log if invalid - * - * valid pipeline: valid pipeline type, modules have unique ordering - * - * valid module: module class exists, module can be loaded, module - * implements correct interface, module has proper methods and modifiers to - * create an instance - * - * @throws IngestModuleLoaderException - */ - private void validate() throws IngestModuleLoaderException { - for (IngestModuleLoader.XmlPipelineRaw pRaw : pipelinesXML) { - boolean pipelineErrors = false; - - //check pipelineType - String pipelineType = pRaw.type; - - IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE pType = null; - - try { - pType = IngestModuleLoader.XmlPipelineRaw.getPipelineType(pipelineType); - } catch (IllegalArgumentException e) { - pipelineErrors = true; - logger.log(Level.SEVERE, "Unknown pipeline type: " + pipelineType); - - } - //ordering store - Map orderings = new HashMap(); - - for (IngestModuleLoader.XmlModuleRaw pMod : pRaw.modules) { - boolean moduleErrors = false; - - //record ordering for validation - int order = pMod.order; - if (orderings.containsKey(order)) { - orderings.put(order, orderings.get(order) + 1); - } else { - orderings.put(order, 1); - } - - //check pipelineType - String modType = pMod.type; - if (!modType.equals(IngestModuleLoader.XmlModuleRaw.MODULE_TYPE.PLUGIN.toString())) { - moduleErrors = true; - logger.log(Level.SEVERE, "Unknown module type: " + modType); - } - - //classes exist and interfaces implemented - String location = pMod.location; - try { - //netbeans uses custom class loader, otherwise can't load classes from other modules - - final Class moduleClass = Class.forName(location, false, classLoader); - final Type intf = moduleClass.getGenericSuperclass(); - - if (pType != null) { - Class moduleMeta = ((IngestModuleMapping) pType).getIngestModuleInterface(); - String moduleIntNameCan = moduleMeta.getCanonicalName(); - String[] moduleIntNameTok = moduleIntNameCan.split(" "); - String moduleIntName = moduleIntNameTok[moduleIntNameTok.length - 1]; - - String intNameCan = intf.toString(); - String[] intNameCanTok = intNameCan.split(" "); - String intName = intNameCanTok[intNameCanTok.length - 1]; - if (!intName.equals(moduleIntName)) { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location - + " does not implement correct interface: " + moduleMeta.getName() - + " required for pipeline: " + pType.toString() - + ", module will not be active."); - } - } else { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location + " does not implement any interface, module will not be active."); - } - - //if file module: check if has public static getDefault() - if (pType == IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.FILE_ANALYSIS) { - try { - Method getDefaultMethod = moduleClass.getMethod("getDefault"); - int modifiers = getDefaultMethod.getModifiers(); - if (!(Modifier.isPublic(modifiers) && Modifier.isStatic(modifiers))) { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location + " does not implement public static getDefault() singleton method."); - } - if (!getDefaultMethod.getReturnType().equals(moduleClass)) { - logger.log(Level.WARNING, "Module class: " + location + " getDefault() singleton method should return the module class instance: " + moduleClass.getName()); - } - - } catch (NoSuchMethodException ex) { - Exceptions.printStackTrace(ex); - } catch (SecurityException ex) { - Exceptions.printStackTrace(ex); - } - } //if data source module: check if has public constructor with no args - else if (pType == IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.DATA_SOURCE_ANALYSIS) { - try { - Constructor constr = moduleClass.getConstructor(); - int modifiers = constr.getModifiers(); - if (!Modifier.isPublic(modifiers)) { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location + " lacks a public default constructor."); - } - } catch (NoSuchMethodException ex) { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location + " lacks a public default constructor."); - } catch (SecurityException ex) { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location + " lacks a public default constructor."); - } - } - - } catch (ClassNotFoundException ex) { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location + " not found, module will not be active."); - - } catch (LinkageError le) { - moduleErrors = true; - logger.log(Level.WARNING, "Module class: " + location + " has unresolved symbols, module will not be active.", le); - } - - - //validate ordering - for (int o : orderings.keySet()) { - int count = orderings.get(o); - if (count > 1) { - pipelineErrors = true; - logger.log(Level.SEVERE, "Pipeline " + pipelineType + " invalid non-unique ordering of modules, order: " + o); - } - } - - pMod.valid = !moduleErrors; - logger.log(Level.INFO, "Module " + pMod.location + " valid: " + pMod.valid); - } //end module - - pRaw.valid = !pipelineErrors; - logger.log(Level.INFO, "Pipeline " + pType + " valid: " + pRaw.valid); - } //end pipeline - - } - - private Set getJarPaths(String modulesDir) { - Set urls = new HashSet(); - - final File modulesDirF = new File(modulesDir); - FilenameFilter jarFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return dir.equals(modulesDirF) && name.endsWith(".jar"); - } - }; - File[] dirJars = modulesDirF.listFiles(jarFilter); - if (dirJars != null) { - //modules dir exists - for (int i = 0; i < dirJars.length; ++i) { - String urlPath = "file:/" + dirJars[i].getAbsolutePath(); - try { - urlPath = URLDecoder.decode(urlPath, ENCODING); - } catch (UnsupportedEncodingException ex) { - logger.log(Level.SEVERE, "Could not decode file path. ", ex); - } - - try { - urls.add(new URL(urlPath)); - //logger.log(Level.INFO, "JAR: " + urlPath); - } catch (MalformedURLException ex) { - logger.log(Level.WARNING, "Invalid URL: " + urlPath, ex); - } - } - } - - /* - * netbeans way, but not public API - org.openide.filesystems.Repository defaultRepository = Repository.getDefault(); - FileSystem masterFilesystem = defaultRepository.getDefaultFileSystem(); - org.netbeans.core.startup.ModuleSystem moduleSystem = new org.netbeans.core.startup.ModuleSystem(masterFilesystem); - List jars = moduleSystem.getModuleJars(); - for (File jar : jars) { - logger.log(Level.INFO, " JAR2: " + jar.getAbsolutePath()); - } - //org.netbeans.ModuleManager moduleManager = moduleSystem.getManager(); - */ - - return urls; - } - - /** - * Get jar paths of autodiscovered modules - * - * @param moduleInfos to look into to discover module jar paths - * @return - */ - private Set getJarPaths(Collection moduleInfos) { - Set urls = new HashSet(); - - //TODO lookup module jar file paths by "seed" class or resource, using the module loader - //problem: we don't have a reliable "seed" class in every moduke - //and loading by Bundle.properties resource does not seem to work with the module class loader - //for now hardcoding jar file locations - - /* - for (ModuleInfo moduleInfo : moduleInfos) { - - if (moduleInfo.isEnabled() == false) { - continue; - } - - String basePackageName = moduleInfo.getCodeNameBase(); - if (basePackageName.startsWith("org.netbeans") - || basePackageName.startsWith("org.openide")) { - //skip - continue; - } - - - ClassLoader moduleClassLoader = moduleInfo.getClassLoader(); - - URL modURL = moduleClassLoader.getResource(basePackageName); - logger.log(Level.INFO, "GOT MOD URL1 : " + modURL); - - modURL = moduleClassLoader.getParent().getResource(basePackageName); - logger.log(Level.INFO, "GOT MOD URL2 : " + modURL); - - modURL = classLoader.getResource(basePackageName); - logger.log(Level.INFO, "GOT MOD URL3 : " + modURL); - } */ - /* - URL modURL = moduleClassLoader.getParent().getResource("Bundle.properties"); - //URL modURL = classLoader.getResource(basePackageName); - logger.log(Level.INFO, "GOT MOD URL : " + modURL); - - modURL = moduleClassLoader.getResource(basePackageName + ".Bundle.properties"); - //URL modURL = classLoader.getResource(basePackageName); - logger.log(Level.INFO, "GOT MOD URL : " + modURL); - - modURL = moduleClassLoader.getResource("Bundle"); - //URL modURL = classLoader.getResource(basePackageName); - logger.log(Level.INFO, "GOT MOD URL : " + modURL); - - Class modClass; - try { - modClass = classLoader.loadClass(basePackageName + ".Installer"); - URL modURL2 = modClass.getProtectionDomain().getCodeSource().getLocation(); - logger.log(Level.INFO, "GOT MOD URL2 : " + modURL2); - } catch (ClassNotFoundException ex) { - // Exceptions.printStackTrace(ex); - } - try { - Class moduleBundleClass = - Class.forName(basePackageName, false, classLoader); - URL modURL3 = moduleBundleClass.getProtectionDomain().getCodeSource().getLocation(); - logger.log(Level.INFO, "GOT MOD URL3 : " + modURL3); - } catch (ClassNotFoundException ex) { - // Exceptions.printStackTrace(ex); - } - - - URL urltry; - try { - urltry = moduleClassLoader.loadClass("Bundle").getProtectionDomain().getCodeSource().getLocation(); - logger.log(Level.INFO, "GOT TRY URL : " + urltry); - } catch (ClassNotFoundException ex) { - // Exceptions.printStackTrace(ex); - } - - } - * */ - - //core modules - urls.addAll(getJarPaths(PlatformUtil.getInstallModulesPath())); - - //user modules - urls.addAll(getJarPaths(PlatformUtil.getUserModulesPath())); - - // add other project dirs, such as from external modules - for (String projectDir : PlatformUtil.getProjectsDirs()) { - File modules = new File(projectDir + File.separator + "modules"); - if (modules.exists()) { - urls.addAll(getJarPaths(modules.getAbsolutePath())); - } - } - - - - return urls; - } - - /** - * Auto-discover ingest modules in all platform modules that are "enabled" - * If discovered ingest module is not already in XML config, add it do - * config and add to in-memory pipeline. - * - * @throws IngestModuleLoaderException - */ - @SuppressWarnings("unchecked") - private void autodiscover() throws IngestModuleLoaderException { - - // Use Lookup to find the other NBM modules. We'll later search them for ingest modules - Collection moduleInfos = Lookup.getDefault().lookupAll(ModuleInfo.class); - logger.log(Level.INFO, "Autodiscovery, found #platform modules: " + moduleInfos.size()); - - Set urls = getJarPaths(moduleInfos); - ArrayList reflectionsSet = new ArrayList<>(); - - for (final ModuleInfo moduleInfo : moduleInfos) { - if (moduleInfo.isEnabled()) { - /* NOTE: We have an assumption here that the modules in an NBM will - * have the same package name as the NBM name. This means that - * an NBM can have only one package with modules in it. */ - String basePackageName = moduleInfo.getCodeNameBase(); - - // skip the standard ones - if (basePackageName.startsWith("org.netbeans") - || basePackageName.startsWith("org.openide")) { - continue; - } - - logger.log(Level.INFO, "Found module: " + moduleInfo.getDisplayName() + " " + basePackageName - + " Build version: " + moduleInfo.getBuildVersion() - + " Spec version: " + moduleInfo.getSpecificationVersion() - + " Impl version: " + moduleInfo.getImplementationVersion()); - - ConfigurationBuilder cb = new ConfigurationBuilder(); - cb.filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix(basePackageName))); - cb.setUrls(urls); - cb.setScanners(new SubTypesScanner(), new ResourcesScanner()); - reflectionsSet.add(new Reflections(cb)); - } - else { - // log if we have our own modules disabled - if (moduleInfo.getCodeNameBase().startsWith("org.sleuthkit")) { - logger.log(Level.WARNING, "Sleuth Kit Module not enabled: " + moduleInfo.getDisplayName()); - } - } - } - - /* This area is used to load the example modules. They are not found via lookup since they - * are in this NBM module. - * Uncomment this section to rum the examples. - */ - /* - ConfigurationBuilder cb = new ConfigurationBuilder(); - cb.filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix("org.sleuthkit.autopsy.examples"))); - cb.setUrls(urls); - cb.setScanners(new SubTypesScanner(), new ResourcesScanner()); - reflectionsSet.add(new Reflections(cb)); - */ - - for (Reflections reflections : reflectionsSet) { - - Set fileModules = reflections.getSubTypesOf(IngestModuleAbstractFile.class); - Iterator it = fileModules.iterator(); - while (it.hasNext()) { - logger.log(Level.INFO, "Found file ingest module in: " + reflections.getClass().getSimpleName() + ": " + it.next().toString()); - } - - Set dataSourceModules = reflections.getSubTypesOf(IngestModuleDataSource.class); - it = dataSourceModules.iterator(); - while (it.hasNext()) { - logger.log(Level.INFO, "Found DataSource ingest module in: " + reflections.getClass().getSimpleName() + ": " + it.next().toString()); - } - - if ((fileModules.isEmpty()) && (dataSourceModules.isEmpty())) { - logger.log(Level.INFO, "Module has no ingest modules: " + reflections.getClass().getSimpleName()); - continue; - } - - //find out which modules to add - //TODO check which modules to remove (which modules were uninstalled) - boolean modulesChanged = false; - - it = fileModules.iterator(); - while (it.hasNext()) { - boolean exists = false; - Class foundClass = (Class) it.next(); - - for (IngestModuleLoader.XmlPipelineRaw rawP : pipelinesXML) { - if (!rawP.type.equals(IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.FILE_ANALYSIS.toString())) { - continue; //skip - } - - for (IngestModuleLoader.XmlModuleRaw rawM : rawP.modules) { - //logger.log(Level.INFO, "CLASS NAME : " + foundClass.getName()); - if (foundClass.getName().equals(rawM.location)) { - exists = true; - break; - } - } - if (exists == true) { - break; - } - } - - if (exists == false) { - logger.log(Level.INFO, "Discovered a new file module to load: " + foundClass.getName()); - //ADD MODULE - addModuleToRawPipeline(foundClass, IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.FILE_ANALYSIS); - modulesChanged = true; - } - - } - - it = dataSourceModules.iterator(); - while (it.hasNext()) { - boolean exists = false; - Class foundClass = (Class) it.next(); - - for (IngestModuleLoader.XmlPipelineRaw rawP : pipelinesXML) { - if (!rawP.type.equals(IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.DATA_SOURCE_ANALYSIS.toString())) { - continue; //skip - } - - - for (IngestModuleLoader.XmlModuleRaw rawM : rawP.modules) { - //logger.log(Level.INFO, "CLASS NAME : " + foundClass.getName()); - if (foundClass.getName().equals(rawM.location)) { - exists = true; - break; - } - } - if (exists == true) { - break; - } - } - - if (exists == false) { - logger.log(Level.INFO, "Discovered a new DataSource module to load: " + foundClass.getName()); - //ADD MODULE - addModuleToRawPipeline(foundClass, IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.DATA_SOURCE_ANALYSIS); - modulesChanged = true; - } - - } - - if (modulesChanged) { - save(); - - try { - pcs.firePropertyChange(IngestModuleLoader.Event.ModulesReloaded.toString(), 0, 1); - } - catch (Exception e) { - logger.log(Level.SEVERE, "IngestModuleLoader listener threw exception", e); - MessageNotifyUtil.Notify.show(NbBundle.getMessage(this.getClass(), "IngestModuleLoader.moduleErr"), - NbBundle.getMessage(this.getClass(), - "IngestModuleLoader.moduleErr.errListenUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); - } - } - - /* - //Enumeration resources = moduleClassLoader.getResources(basePackageName); - Enumeration resources = classLoader.getResources(basePackageName); - while (resources.hasMoreElements()) { - System.out.println(resources.nextElement()); - } */ - } - } - - /** - * Set a new order of the module - * - * @param pipeLineType pipeline type where the module to reorder is present - * @param moduleLocation loaded module name (location), fully qualified - * class path - * @param newOrder new order to set - */ - void setModuleOrder(IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE pipeLineType, String moduleLocation, int newOrder) throws IngestModuleLoaderException { - throw new IngestModuleLoaderException( - NbBundle.getMessage(this.getClass(), "IngestModuleLoader.exception.notImplemented.msg")); - } - - /** - * add autodiscovered module to raw pipeline to be validated and - * instantiated - * - * @param moduleClass - * @param pipelineType - */ - private void addModuleToRawPipeline(Class moduleClass, IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE pipelineType) throws IngestModuleLoaderException { - String moduleLocation = moduleClass.getName(); - - IngestModuleLoader.XmlModuleRaw modRaw = new IngestModuleLoader.XmlModuleRaw(); - modRaw.arguments = ""; //default, no arguments - modRaw.location = moduleLocation; - modRaw.order = Integer.MAX_VALUE - (numModDiscovered++); //add to end - modRaw.type = IngestModuleLoader.XmlModuleRaw.MODULE_TYPE.PLUGIN.toString(); - modRaw.valid = false; //to be validated - - //save the current numModDiscovered - ModuleSettings.setConfigSetting(IngestManager.MODULE_PROPERTIES, CUR_MODULES_DISCOVERED_SETTING, Integer.toString(numModDiscovered)); - - //find the pipeline of that type - IngestModuleLoader.XmlPipelineRaw pipeline = null; - for (IngestModuleLoader.XmlPipelineRaw rawP : this.pipelinesXML) { - if (rawP.type.equals(pipelineType.toString())) { - pipeline = rawP; - break; - } - } - if (pipeline == null) { - throw new IngestModuleLoaderException( - NbBundle.getMessage(this.getClass(), "IngestModuleLoader.exception.cantFindPipeline.msg", - pipelineType.toString(), moduleLocation)); - } else { - pipeline.modules.add(modRaw); - logger.log(Level.INFO, "Added a new module " + moduleClass.getName() + " to pipeline " + pipelineType.toString()); - } - } - - /** - * Register a listener for module install/uninstall //TODO ensure that - * module is actually loadable when Lookup event is fired - */ - private void registerModulesChange() { - final Lookup.Result result = - Lookup.getDefault().lookupResult(ModuleInfo.class); - result.addLookupListener(new LookupListener() { - @Override - public void resultChanged(LookupEvent event) { - try { - logger.log(Level.INFO, "Module change occured, reloading."); - init(); - } catch (IngestModuleLoaderException ex) { - logger.log(Level.SEVERE, "Error reloading the module loader. ", ex); - } - } - }); - } - - /** - * Save the current in memory pipeline config, including autodiscovered - * modules - * - * @throws IngestModuleLoaderException - */ - public void save() throws IngestModuleLoaderException { - DocumentBuilderFactory dbfac = DocumentBuilderFactory.newInstance(); - - try { - DocumentBuilder docBuilder = dbfac.newDocumentBuilder(); - Document doc = docBuilder.newDocument(); - - - Comment comment = doc.createComment( - NbBundle.getMessage(this.getClass(), "IngestModuleLoader.save.comment.text", getClass().getName(), - dateFormatter.format(System.currentTimeMillis()))); - doc.appendChild(comment); - Element rootEl = doc.createElement(IngestModuleLoader.XmlPipelineRaw.XML_PIPELINE_ROOT); - doc.appendChild(rootEl); - - for (IngestModuleLoader.XmlPipelineRaw rawP : this.pipelinesXML) { - Element pipelineEl = doc.createElement(IngestModuleLoader.XmlPipelineRaw.XML_PIPELINE_EL); - pipelineEl.setAttribute(IngestModuleLoader.XmlPipelineRaw.XML_PIPELINE_TYPE_ATTR, rawP.type); - rootEl.appendChild(pipelineEl); - - for (IngestModuleLoader.XmlModuleRaw rawM : rawP.modules) { - Element moduleEl = doc.createElement(IngestModuleLoader.XmlModuleRaw.XML_MODULE_EL); - - moduleEl.setAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_LOC_ATTR, rawM.location); - moduleEl.setAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_TYPE_ATTR, rawM.type); - moduleEl.setAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_ORDER_ATTR, Integer.toString(rawM.order)); - moduleEl.setAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_TYPE_ATTR, rawM.type); - - pipelineEl.appendChild(moduleEl); - } - } - - XMLUtil.saveDoc(IngestModuleLoader.class, absFilePath, ENCODING, doc); - logger.log(Level.INFO, "Pipeline configuration saved to: " + this.absFilePath); - } catch (ParserConfigurationException e) { - logger.log(Level.SEVERE, "Error saving pipeline config XML: can't initialize parser.", e); - } - - } - - /** - * Instantiate valid pipeline and modules and store the module object - * references - * - * @throws IngestModuleLoaderException - */ - @SuppressWarnings("unchecked") - private void instantiate() throws IngestModuleLoaderException { - - //clear current - filePipeline.clear(); - dataSourcePipeline.clear(); - - //add autodiscovered modules to pipelinesXML - autodiscover(); - - //validate all modules: from XML + just autodiscovered - - validate(); - - for (IngestModuleLoader.XmlPipelineRaw pRaw : pipelinesXML) { - if (pRaw.valid == false) { - //skip invalid pipelines - continue; - } - - //sort modules by order parameter, in case XML order is different - Collections.sort(pRaw.modules, new Comparator() { - @Override - public int compare(IngestModuleLoader.XmlModuleRaw o1, IngestModuleLoader.XmlModuleRaw o2) { - return Integer.valueOf(o1.order).compareTo(Integer.valueOf(o2.order)); - } - }); - - //check pipelineType, add to right pipeline collection - IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE pType = IngestModuleLoader.XmlPipelineRaw.getPipelineType(pRaw.type); - - for (IngestModuleLoader.XmlModuleRaw pMod : pRaw.modules) { - try { - if (pMod.valid == false) { - //skip invalid modules - continue; - } - - //add to right pipeline - switch (pType) { - case FILE_ANALYSIS: - IngestModuleAbstractFile fileModuleInstance = null; - final Class fileModuleClass = - (Class) Class.forName(pMod.location, true, classLoader); - try { - Method getDefaultMethod = fileModuleClass.getMethod("getDefault"); - if (getDefaultMethod != null) { - fileModuleInstance = (IngestModuleAbstractFile) getDefaultMethod.invoke(null); - } - } catch (NoSuchMethodException ex) { - logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); - pMod.valid = false; //prevent from trying to load again - } catch (SecurityException ex) { - logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); - pMod.valid = false; //prevent from trying to load again - } catch (IllegalAccessException ex) { - logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); - pMod.valid = false; //prevent from trying to load again - } catch (InvocationTargetException ex) { - logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); - pMod.valid = false; //prevent from trying to load again - } - - filePipeline.add(fileModuleInstance); - break; - case DATA_SOURCE_ANALYSIS: - final Class dataSourceModuleClass = - (Class) Class.forName(pMod.location, true, classLoader); - - try { - Constructor constr = dataSourceModuleClass.getConstructor(); - IngestModuleDataSource dataSourceModuleInstance = constr.newInstance(); - - if (dataSourceModuleInstance != null) { - dataSourcePipeline.add(dataSourceModuleInstance); - } - - } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) { - logger.log(Level.WARNING, "Validated module, could not initialize, check for bugs in the module: " + pMod.location, ex); - pMod.valid = false; - } - - - break; - default: - logger.log(Level.SEVERE, "Unexpected pipeline type to add module to: " + pType); - } - - - } catch (ClassNotFoundException ex) { - logger.log(Level.SEVERE, "Validated module, but could not load (shouldn't happen): " + pMod.location); - } - } - - } //end instantiating modules in XML - - - - } - - /** - * Get a new instance of the module or null if could not be created - * - * @param module existing module to get an instance of - * @return new module instance or null if could not be created - */ - IngestModuleAbstract getNewIngestModuleInstance(IngestModuleAbstract module) { - try { - IngestModuleAbstract newInstance = module.getClass().newInstance(); - return newInstance; - } catch (InstantiationException e) { - logger.log(Level.SEVERE, "Cannot instantiate module: " + module.getName(), e); - return null; - } catch (IllegalAccessException e) { - logger.log(Level.SEVERE, "Cannot instantiate module: " + module.getName(), e); - return null; - } - - } - - private IngestModuleAbstract getNewIngestModuleInstance(Class moduleClass) { - try { - IngestModuleAbstract newInstance = moduleClass.newInstance(); - return newInstance; - } catch (InstantiationException e) { - logger.log(Level.SEVERE, "Cannot instantiate module: " + moduleClass.getName(), e); - return null; - } catch (IllegalAccessException e) { - logger.log(Level.SEVERE, "Cannot instantiate module: " + moduleClass.getName(), e); - return null; - } - - } - - /** - * Load XML into raw pipeline representation - * - * @throws IngestModuleLoaderException - */ - private void loadRawPipeline() throws IngestModuleLoaderException { - final Document doc = XMLUtil.loadDoc(IngestModuleLoader.class, absFilePath, XSDFILE); - if (doc == null) { - throw new IngestModuleLoaderException( - NbBundle.getMessage(this.getClass(), "IngestModuleLoader.loadRawPipeline.exception.cantLoadXML.msg", - this.absFilePath)); - } - Element root = doc.getDocumentElement(); - if (root == null) { - String msg = NbBundle - .getMessage(this.getClass(), "IngestModuleLoader.loadRawPipeline.exception.invalidFileFormat.msg"); - logger.log(Level.SEVERE, msg); - throw new IngestModuleLoaderException(msg); - } - NodeList pipelineNodes = root.getElementsByTagName(IngestModuleLoader.XmlPipelineRaw.XML_PIPELINE_EL); - int numPipelines = pipelineNodes.getLength(); - if (numPipelines == 0) { - throw new IngestModuleLoaderException(NbBundle.getMessage(this.getClass(), - "IngestModuleLoader.loadRawPipeline.exception.noPipelinesInConf.msg", - absFilePath)); - } - for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) { - //process pipelines - Element pipelineEl = (Element) pipelineNodes.item(pipelineNum); - final String pipelineType = pipelineEl.getAttribute(IngestModuleLoader.XmlPipelineRaw.XML_PIPELINE_TYPE_ATTR); - logger.log(Level.INFO, "Found pipeline type: " + pipelineType); - - IngestModuleLoader.XmlPipelineRaw pipelineRaw = new IngestModuleLoader.XmlPipelineRaw(); - pipelineRaw.type = pipelineType; - this.pipelinesXML.add(pipelineRaw); - - //process modules - NodeList modulesNodes = pipelineEl.getElementsByTagName(IngestModuleLoader.XmlModuleRaw.XML_MODULE_EL); - int numModules = modulesNodes.getLength(); - if (numModules == 0) { - logger.log(Level.WARNING, "Pipeline: " + pipelineType + " has no modules defined."); - } - for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) { - //process modules - Element moduleEl = (Element) modulesNodes.item(moduleNum); - final String moduleType = moduleEl.getAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_TYPE_ATTR); - final String moduleOrder = moduleEl.getAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_ORDER_ATTR); - final String moduleLoc = moduleEl.getAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_LOC_ATTR); - final String moduleArgs = moduleEl.getAttribute(IngestModuleLoader.XmlModuleRaw.XML_MODULE_ARGS_ATTR); - IngestModuleLoader.XmlModuleRaw module = new IngestModuleLoader.XmlModuleRaw(); - module.arguments = moduleArgs; - module.location = moduleLoc; - try { - module.order = Integer.parseInt(moduleOrder); - } catch (NumberFormatException e) { - logger.log(Level.WARNING, "Invalid module order, need integer: " + moduleOrder + ", adding to end of the list"); - module.order = Integer.MAX_VALUE - (numModDiscovered++); - //save the current numModDiscovered - ModuleSettings.setConfigSetting(IngestManager.MODULE_PROPERTIES, CUR_MODULES_DISCOVERED_SETTING, Integer.toString(numModDiscovered)); - - } - module.type = moduleType; - pipelineRaw.modules.add(module); - } - - } - - } - - /** - * Load and validate XML pipeline, autodiscover and instantiate the pipeline - * modules Can be called multiple times to refresh the view of modules - * - * @throws IngestModuleLoaderException - */ - public synchronized void init() throws IngestModuleLoaderException { - absFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINE_CONFIG_XML; - ClassLoader parentClassLoader = Lookup.getDefault().lookup(ClassLoader.class); - classLoader = new CustomClassLoader(parentClassLoader); - - try { - boolean extracted = PlatformUtil.extractResourceToUserConfigDir(IngestModuleLoader.class, PIPELINE_CONFIG_XML); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir ", ex); - } - - //load the pipeline config - loadRawPipeline(); - - instantiate(); - - - } - - /** - * Get loaded file modules - * - * @return file modules loaded - */ - public List getAbstractFileIngestModules() { - return filePipeline; - } - - /** - * Get loaded data source modules - * - * @return data source modules loaded - */ - public List getDataSourceIngestModules() { - return dataSourcePipeline; - } - - //pipeline XML representation - private static final class XmlPipelineRaw { - - enum PIPELINE_TYPE implements IngestModuleMapping { - - FILE_ANALYSIS { - @Override - public String toString() { - return "FileAnalysis"; - } - - @Override - public Class getIngestModuleInterface() { - return IngestModuleAbstractFile.class; - } - }, - DATA_SOURCE_ANALYSIS { - @Override - public String toString() { - return "ImageAnalysis"; - } - - @Override - public Class getIngestModuleInterface() { - return IngestModuleDataSource.class; - } - },; - } - - /** - * get pipeline type for string mapping to type toString() method - * - * @param s string equals to one of the types toString() representation - * @return matching type - */ - static IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE getPipelineType(String s) throws IllegalArgumentException { - IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE[] types = IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.values(); - for (int i = 0; i < types.length; ++i) { - if (types[i].toString().equals(s)) { - return types[i]; - } - } - throw new IllegalArgumentException( - NbBundle.getMessage(IngestModuleLoader.class, "IngestModuleLoader.exception.noPipelineTypeForStr.msg", s)); - } - private static final String XML_PIPELINE_ROOT = "PIPELINE_CONFIG"; - private static final String XML_PIPELINE_EL = "PIPELINE"; - private static final String XML_PIPELINE_TYPE_ATTR = "type"; - String type; - List modules = new ArrayList(); - boolean valid = false; // if passed validation - } - - private static final class XmlModuleRaw { - - enum MODULE_TYPE { - - PLUGIN { - @Override - public String toString() { - return "plugin"; - } - },; - } - //XML tags and attributes - private static final String XML_MODULE_EL = "MODULE"; - private static final String XML_MODULE_ORDER_ATTR = "order"; - private static final String XML_MODULE_TYPE_ATTR = "type"; - private static final String XML_MODULE_LOC_ATTR = "location"; - private static final String XML_MODULE_ARGS_ATTR = "arguments"; - int order; - String type; - String location; - String arguments; - boolean valid = false; // if passed validation - } -} - -/** - * Exception thrown when errors occur while loading modules - */ -class IngestModuleLoaderException extends Throwable { - - public IngestModuleLoaderException(String message) { - super(message); - } - - public IngestModuleLoaderException(String message, Throwable cause) { - super(message, cause); - } -} - -/** - * Implements mapping of a type to ingest module interface type - */ -interface IngestModuleMapping { - - /** - * Get ingest module interface mapped to that type - * - * @return ingest module interface meta type - */ - public Class getIngestModuleInterface(); -} - -/** - * Custom class loader that attempts to force class resolution / linkage validation at loading - */ -class CustomClassLoader extends ClassLoader { - private static final Logger logger = Logger.getLogger(CustomClassLoader.class.getName()); - - CustomClassLoader(ClassLoader parent) { - super(parent); - } - - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - logger.log(Level.INFO, "Custom loading class: " + name); - - Class cl = super.loadClass(name, true); - - return cl; - } - -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleSettings.java new file mode 100755 index 0000000000..b55b0341c6 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleSettings.java @@ -0,0 +1,29 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.io.Serializable; + +/** + * Interface for per ingest job options for ingest modules. Options are + * serializable to support persistence of options between invocations of the + * application. + */ +public interface IngestModuleSettings extends Serializable { +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleSettingsPanel.java new file mode 100755 index 0000000000..d4fa7b32d9 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleSettingsPanel.java @@ -0,0 +1,34 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import javax.swing.JPanel; + +/** + * Abstract base class for ingest module job settings panels. + */ +public abstract class IngestModuleSettingsPanel extends JPanel { + + /** + * Gets the ingest job settings for an ingest module. + * + * @return The ingest settings. + */ + public abstract IngestModuleSettings getSettings(); +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java new file mode 100755 index 0000000000..b37c998e07 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java @@ -0,0 +1,91 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +/** + * Combines an ingest module factory with ingest module settings and an enabled + * flag to create a template for creating fully configured ingest modules. + */ +final class IngestModuleTemplate { + + private final IngestModuleFactory moduleFactory; + private IngestModuleSettings settings = null; + private boolean enabled = true; + + IngestModuleTemplate(IngestModuleFactory moduleFactory, IngestModuleSettings settings) { + this.moduleFactory = moduleFactory; + this.settings = settings; + } + + String getModuleName() { + return moduleFactory.getModuleDisplayName(); + } + + String getModuleDescription() { + return moduleFactory.getModuleDescription(); + } + + IngestModuleSettings getModuleSettings() { + return settings; + } + + void setModuleSettings(IngestModuleSettings settings) { + this.settings = settings; + } + + boolean hasModuleSettingsPanel() { + return moduleFactory.hasModuleSettingsPanel(); + } + + IngestModuleSettingsPanel getModuleSettingsPanel() { + return moduleFactory.getModuleSettingsPanel(settings); + } + + boolean hasGlobalSettingsPanel() { + return moduleFactory.hasGlobalSettingsPanel(); + } + + IngestModuleGlobalSetttingsPanel getGlobalSettingsPanel() { + return moduleFactory.getGlobalSettingsPanel(); + } + + boolean isDataSourceIngestModuleTemplate() { + return moduleFactory.isDataSourceIngestModuleFactory(); + } + + DataSourceIngestModule createDataSourceIngestModule() { + return moduleFactory.createDataSourceIngestModule(settings); + } + + boolean isFileIngestModuleTemplate() { + return moduleFactory.isFileIngestModuleFactory(); + } + + FileIngestModule createFileIngestModule() { + return moduleFactory.createFileIngestModule(settings); + } + + void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + boolean isEnabled() { + return enabled; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index c97875d69d..ba7d21428f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -38,14 +38,14 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; /** * Monitor health of the system and stop ingest if necessary */ - public class IngestMonitor { +public final class IngestMonitor { + public static final int DISK_FREE_SPACE_UNKNOWN = -1; private static final int INITIAL_INTERVAL_MS = 60000; //1 min. private final Logger logger = Logger.getLogger(IngestMonitor.class.getName()); private Timer timer; private static final java.util.logging.Logger MONITOR_LOGGER = java.util.logging.Logger.getLogger("monitor"); private MonitorAction monitor; - public static final int DISK_FREE_SPACE_UNKNOWN = -1; IngestMonitor() { @@ -166,7 +166,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath); logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath); manager.stopAll(); - manager.postMessage(IngestMessage.createManagerErrorMessage( + IngestServices.getDefault().postMessage(IngestMessage.createManagerErrorMessage( NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath), NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath))); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java new file mode 100755 index 0000000000..5c93b9d102 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java @@ -0,0 +1,137 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +/** + * Provides data source and file ingest pipeline configurations as ordered lists + * of ingest module class names. The order of the module class names indicates + * the desired sequence of ingest module instances in an ingest modules + * pipeline. + */ +final class IngestPipelinesConfiguration { + + private static final Logger logger = Logger.getLogger(IngestPipelinesConfiguration.class.getName()); + private final static String PIPELINES_CONFIG_FILE = "pipeline_config.xml"; + private final static String PIPELINES_CONFIG_FILE_XSD = "PipelineConfigSchema.xsd"; + private static final String XML_PIPELINE_ELEM = "PIPELINE"; + private static final String XML_PIPELINE_TYPE_ATTR = "type"; + private final static String DATA_SOURCE_INGEST_PIPELINE_TYPE = "ImageAnalysis"; + private final static String FILE_INGEST_PIPELINE_TYPE = "FileAnalysis"; + private static final String XML_MODULE_ELEM = "MODULE"; + private static final String XML_MODULE_CLASS_NAME_ATTR = "location"; + private static IngestPipelinesConfiguration instance; + private final List dataSourceIngestPipelineConfig = new ArrayList<>(); + private final List fileIngestPipelineConfig = new ArrayList<>(); + + private IngestPipelinesConfiguration() { + readPipelinesConfigurationFile(); + } + + synchronized static IngestPipelinesConfiguration getInstance() { + if (instance == null) { + Logger.getLogger(IngestPipelinesConfiguration.class.getName()).log(Level.INFO, "Creating ingest module loader instance"); + instance = new IngestPipelinesConfiguration(); + } + return instance; + } + + List getDataSourceIngestPipelineConfig() { + return new ArrayList<>(dataSourceIngestPipelineConfig); + } + + List getFileIngestPipelineConfig() { + return new ArrayList<>(fileIngestPipelineConfig); + } + + private void readPipelinesConfigurationFile() { + try { + PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE); + } catch (IOException ex) { + logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir", ex); + return; + } + + String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE; + Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath, PIPELINES_CONFIG_FILE_XSD); + if (doc == null) { + return; + } + + Element rootElement = doc.getDocumentElement(); + if (rootElement == null) { + logger.log(Level.SEVERE, "Invalid pipelines config file"); + return; + } + + NodeList pipelineElements = rootElement.getElementsByTagName(XML_PIPELINE_ELEM); + int numPipelines = pipelineElements.getLength(); + if (numPipelines < 1 || numPipelines > 2) { + logger.log(Level.SEVERE, "Invalid pipelines config file"); + return; + } + + List pipelineConfig = null; + for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) { + Element pipelineElement = (Element) pipelineElements.item(pipelineNum); + String pipelineTypeAttr = pipelineElement.getAttribute(XML_PIPELINE_TYPE_ATTR); + if (pipelineTypeAttr != null) { + switch (pipelineTypeAttr) { + case DATA_SOURCE_INGEST_PIPELINE_TYPE: + pipelineConfig = dataSourceIngestPipelineConfig; + break; + case FILE_INGEST_PIPELINE_TYPE: + pipelineConfig = fileIngestPipelineConfig; + break; + default: + logger.log(Level.SEVERE, "Invalid pipelines config file"); + return; + } + } + + // Create an ordered list of class names. The sequence of class + // names defines the sequence of modules in the pipeline. + if (pipelineConfig != null) { + NodeList modulesElems = pipelineElement.getElementsByTagName(XML_MODULE_ELEM); + int numModules = modulesElems.getLength(); + if (numModules == 0) { + break; + } + for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) { + Element moduleElement = (Element) modulesElems.item(moduleNum); + final String moduleClassName = moduleElement.getAttribute(XML_MODULE_CLASS_NAME_ATTR); + if (moduleClassName != null) { + pipelineConfig.add(moduleClassName); + } + } + } + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 0989fc30aa..ec3c236cad 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -31,9 +31,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; - -import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.ingest.IngestScheduler.FileScheduler.FileTask; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentVisitor; @@ -51,21 +50,21 @@ import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM; import org.sleuthkit.datamodel.TskData.TSK_FS_META_TYPE_ENUM; /** - * Schedules data source (images, file-sets, etc) and files with their associated modules for ingest, and - * manage queues of the scheduled tasks. + * Schedules data source (images, file-sets, etc) and files with their + * associated modules for ingest, and manage queues of the scheduled tasks. * * Currently a singleton object only (as there is one pipeline at a time) * - * Contains internal schedulers for content objects into data source and and file ingest - * pipelines. + * Contains internal schedulers for content objects into data source and and + * file ingest pipelines. * */ -class IngestScheduler { +final class IngestScheduler { private static IngestScheduler instance; private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName()); private final DataSourceScheduler dataSourceScheduler = new DataSourceScheduler(); private final FileScheduler fileScheduler = new FileScheduler(); - + private IngestScheduler() { } @@ -102,64 +101,125 @@ class IngestScheduler { * */ static class FileScheduler implements Iterator { - //root folders enqueued - private TreeSet rootProcessTasks; + //root folders enqueued + private TreeSet rootDirectoryTasks; //stack of current dirs to be processed recursively - private List curDirProcessTasks; + private List directoryTasks; //list of files being processed in the currently processed directory - private LinkedList curFileProcessTasks; //need to add to start and end quickly + private LinkedList fileTasks; //need to add to start and end quickly - //estimated total files to be enqueued for currently scheduled content objects - private int filesEnqueuedEst; - private int filesDequeued; - private final static int FAT_NTFS_FLAGS = - TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() + //estimated total files to be enqueued for currently scheduled content objects + private int filesEnqueuedEst = 0; + private int filesDequeued = 0; + private final static int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); private FileScheduler() { - rootProcessTasks = new TreeSet<>(new RootTaskComparator()); - curDirProcessTasks = new ArrayList<>(); - curFileProcessTasks = new LinkedList<>(); - resetCounters(); + rootDirectoryTasks = new TreeSet<>(new RootTaskComparator()); + directoryTasks = new ArrayList<>(); + fileTasks = new LinkedList<>(); + resetCounters(); } private void resetCounters() { filesEnqueuedEst = 0; filesDequeued = 0; - } - + } + @Override public synchronized String toString() { StringBuilder sb = new StringBuilder(); - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.rootDirs.text", - rootProcessTasks.size())); - for (FileTask task : rootProcessTasks) { + sb.append("\nRootDirs(sorted), size: ").append(rootDirectoryTasks.size()); + for (FileTask task : rootDirectoryTasks) { sb.append(task.toString()).append(" "); } - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curDirs.text", - curDirProcessTasks.size())); - for (FileTask task : curDirProcessTasks) { + sb.append("\nCurDirs(stack), size: ").append(directoryTasks.size()); + for (FileTask task : directoryTasks) { sb.append(task.toString()).append(" "); } - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curFiles.text", - curFileProcessTasks.size())); - for (FileTask task : curFileProcessTasks) { + sb.append("\nCurFiles, size: ").append(fileTasks.size()); + for (FileTask task : fileTasks) { sb.append(task.toString()).append(" "); } return sb.toString(); } + synchronized void scheduleIngestOfFiles(IngestJob dataSourceTask) { + Content dataSource = dataSourceTask.getDataSource(); + Collection rootObjects = dataSource.accept(new GetRootDirVisitor()); + List firstLevelFiles = new ArrayList<>(); + if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { + // The data source is file. + firstLevelFiles.add((AbstractFile)dataSource); + } + else { + for (AbstractFile root : rootObjects) { + List children; + try { + children = root.getChildren(); + if (children.isEmpty()) { + //add the root itself, could be unalloc file, child of volume or image + firstLevelFiles.add(root); + } + else { + //root for fs root dir, schedule children dirs/files + for (Content child : children) { + if (child instanceof AbstractFile) { + firstLevelFiles.add((AbstractFile) child); + } + } + } + } + catch (TskCoreException ex) { + logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); + } + } + } + + for (AbstractFile firstLevelFile : firstLevelFiles) { + FileTask fileTask = new FileTask(firstLevelFile, dataSourceTask); + if (shouldEnqueueTask(fileTask)) { + rootDirectoryTasks.add(fileTask); + } + } + + // Update approx count of files to process in queues + filesEnqueuedEst = queryNumFilesinEnqueuedContents(); + + // Reshuffle/update the dir and file level queues if needed + updateQueues(); + } + + /** + * Schedule a file to the file ingest, with associated modules. This + * will add the file to beginning of the file queue. The method is + * intended for rescheduling a file that is a derivative of another + * content object that has already ingested and produced this file. As a + * result, the derivative file will be scheduled with the same priority + * as the parent origin file. + * + * @param file file to be scheduled + * @param originalContext original content schedule context that was used + * to schedule the parent origin content, with the modules, settings, etc. + */ + synchronized void scheduleFile(IngestJob ingestJob, AbstractFile file) { + FileTask fileTask = new FileTask(file, ingestJob); + if (shouldEnqueueTask(fileTask)) { + fileTasks.addFirst(fileTask); + fileTasks.add(fileTask); + ++filesEnqueuedEst; + } + } + float getPercentageDone() { if (filesEnqueuedEst == 0) { return 0; } - return ((100.f) * filesDequeued) / filesEnqueuedEst; - } /** @@ -203,230 +263,12 @@ class IngestScheduler { int getFilesDequeued() { return filesDequeued; } - - /** - * Task for a specific file to process. More specific than the - * higher-level DataSourceTask. - */ - static class FileTask { - private final AbstractFile file; - private final DataSourceTask dataSourceTask; - - public FileTask(AbstractFile file, DataSourceTask dataSourceTask) { - this.file = file; - this.dataSourceTask = dataSourceTask; - } - - public DataSourceTask getDataSourceTask() { - return dataSourceTask; - } - - public AbstractFile getFile() { - return file; - } - - @Override - public String toString() { - try { - return NbBundle.getMessage(this.getClass(), "IngestScheduler.fileTask.toString.long", file.getId(), - file.getUniquePath()); // + ", dataSourceTask=" + dataSourceTask + '}'; - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); - } - return NbBundle.getMessage(this.getClass(), "IngestScheduler.fileTask.toString.short", file.getId(), - file.getName()); - } - - /** - * two process tasks are equal when the file/dir and modules are the - * same this enables are not to queue up the same file/dir, modules - * tuples into the root dir set - * - * @param obj - * @return - */ - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final FileTask other = (FileTask) obj; - if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { - return false; - } - DataSourceTask thisTask = this.getDataSourceTask(); - DataSourceTask otherTask = other.getDataSourceTask(); - - if (thisTask != otherTask - && (thisTask == null || !thisTask.equals(otherTask))) { - return false; - } - return true; - } - - - /** - * Create 1 or more ProcessTasks for each root dir in the Content from - * the context supplied - * - * @param context the original ingest context - * @return - */ - private static List createFromScheduledTask(DataSourceTask scheduledTask) { - final Content scheduledContent = scheduledTask.getContent(); - Collection rootObjects = scheduledContent.accept(new GetRootDirVisitor()); - List firstLevelFiles = new ArrayList<>(); - if (rootObjects.isEmpty() && scheduledContent instanceof AbstractFile) { - //add the root, which is a leaf itself - firstLevelFiles.add((AbstractFile) scheduledContent); - } else { - for (AbstractFile root : rootObjects) { - //TODO the type-specific AbstractFile getChildren() method - List children; - try { - children = root.getChildren(); - if (children.isEmpty()) { - //add the root itself, could be unalloc file, child of volume or image - firstLevelFiles.add(root); - } else { - //root for fs root dir, schedule children dirs/files - for (Content child : children) { - if (child instanceof AbstractFile) { - firstLevelFiles.add((AbstractFile) child); - } - } - } - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not get children of root to enqueue: " - + root.getId() + ": " + root.getName(), ex); - } - - } - } - - List processTasks = new ArrayList<>(); - for (AbstractFile firstLevelFile : firstLevelFiles) { - FileTask newTask = new FileTask(firstLevelFile, scheduledTask); - if (shouldEnqueueTask(newTask)) { - processTasks.add(newTask); - } - } - return processTasks; - } - } - - /** - * Remove duplicated tasks from previous ingest enqueue currently it - * removes all previous tasks scheduled in queues for this Content - * - * @param task tasks similar to this one should be removed - */ - private void removeDupTasks(DataSourceTask task) { - final Content inputContent = task.getContent(); - - //remove from root queue - List toRemove = new ArrayList<>(); - for (FileTask pt : rootProcessTasks) { - if (pt.getDataSourceTask().getContent().equals(inputContent)) { - toRemove.add(pt); - } - } - rootProcessTasks.removeAll(toRemove); - - //remove from dir stack - toRemove = new ArrayList<>(); - for (FileTask pt : curDirProcessTasks) { - if (pt.getDataSourceTask().getContent().equals(inputContent)) { - toRemove.add(pt); - } - } - curDirProcessTasks.removeAll(toRemove); - - //remove from file queue - toRemove = new ArrayList<>(); - for (FileTask pt : curFileProcessTasks) { - if (pt.getDataSourceTask().getContent().equals(inputContent)) { - toRemove.add(pt); - } - } - curFileProcessTasks.removeAll(toRemove); - - - } - - /** - * Schedule a file to the file ingest, with associated modules. This - * will add the file to beginning of the file queue. The method is - * intended for rescheduling a file that is a derivative of another - * content object that has already ingested and produced this file. As a - * result, the derivative file will be scheduled with the same priority - * as the parent origin file. - * - * @param file file to be scheduled - * @param originalContext original content schedule context that was used - * to schedule the parent origin content, with the modules, settings, etc. - */ - synchronized void schedule(AbstractFile file, PipelineContext originalContext) { - DataSourceTask originalTask = originalContext.getDataSourceTask(); - - //skip if task contains no modules - if (originalTask.getModules().isEmpty()) { - return; - } - - FileTask fileTask = new FileTask(file, originalContext.getDataSourceTask()); - if (shouldEnqueueTask(fileTask)) { - this.curFileProcessTasks.addFirst(fileTask); - ++filesEnqueuedEst; - } - - - } - - /** - * Schedule new Content object for a file ingest with associated - * modules. - * - * @param context context to schedule, with scheduled task containing content to process and modules - */ - synchronized void schedule(DataSourceTask task) { - //skip if task contains no modules - if (task.getModules().isEmpty()) { - return; - } - - final Content contentToSchedule = task.getContent(); - - if (getSourceContent().contains(contentToSchedule)) { - //reset counters if the same content enqueued twice - //Note, not very accurate, because we may have processed some files from - //another content - this.filesDequeued = 0; - } - - //remove duplicate scheduled tasks still in queues for this content if enqueued previously - removeDupTasks(task); - - List rootTasks = FileTask.createFromScheduledTask(task); - - //adds and resorts the tasks - this.rootProcessTasks.addAll(rootTasks); - - //update approx count of files to process in queues - this.filesEnqueuedEst = this.queryNumFilesinEnqueuedContents(); - - //reshuffle/update the dir and file level queues if needed - updateQueues(); - - } - + @Override public synchronized boolean hasNext() { - if (curFileProcessTasks.isEmpty()) { - resetCounters(); + if (fileTasks.isEmpty()) { + filesEnqueuedEst = 0; + filesDequeued = 0; return false; } return true; @@ -435,12 +277,11 @@ class IngestScheduler { @Override public synchronized FileTask next() { if (!hasNext()) { - throw new IllegalStateException( - NbBundle.getMessage(this.getClass(), "IngestScheduler.FileTask.next.exception.msg")); + throw new IllegalStateException("No next ProcessTask, check hasNext() first!"); } //dequeue the last in the list - final FileTask task = curFileProcessTasks.pollLast(); + final FileTask task = fileTasks.pollLast(); filesDequeued++; updateQueues(); @@ -457,28 +298,28 @@ class IngestScheduler { // that do not get enqueued while (true) { // There are files in the queue, we're done - if (this.curFileProcessTasks.isEmpty() == false) { + if (this.fileTasks.isEmpty() == false) { return; } // fill in the directory queue if it is empty. - if (this.curDirProcessTasks.isEmpty()) { + if (this.directoryTasks.isEmpty()) { // bail out if root is also empty -- we are done - if (rootProcessTasks.isEmpty()) { + if (rootDirectoryTasks.isEmpty()) { return; } - FileTask rootTask = this.rootProcessTasks.pollFirst(); - curDirProcessTasks.add(rootTask); + FileTask rootTask = this.rootDirectoryTasks.pollFirst(); + directoryTasks.add(rootTask); } //pop and push AbstractFile directory children if any //add the popped and its leaf children onto cur file list - FileTask parentTask = curDirProcessTasks.remove(curDirProcessTasks.size() - 1); + FileTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); final AbstractFile parentFile = parentTask.file; // add itself to the file list if (shouldEnqueueTask(parentTask)) { - this.curFileProcessTasks.addLast(parentTask); + this.fileTasks.addLast(parentTask); } // add its children to the file and directory lists @@ -487,13 +328,13 @@ class IngestScheduler { for (Content c : children) { if (c instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) c; - FileTask childTask = new FileTask(childFile, parentTask.getDataSourceTask()); + FileTask childTask = new FileTask(childFile, parentTask.getJob()); if (childFile.hasChildren()) { - this.curDirProcessTasks.add(childTask); + this.directoryTasks.add(childTask); } else if (shouldEnqueueTask(childTask)) { - this.curFileProcessTasks.addLast(childTask); + this.fileTasks.addLast(childTask); } } } @@ -506,8 +347,7 @@ class IngestScheduler { @Override public void remove() { - throw new UnsupportedOperationException( - NbBundle.getMessage(this.getClass(), "IngestScheduler.remove.exception.notSupported.msg")); + throw new UnsupportedOperationException("Not supported."); } /** @@ -520,61 +360,25 @@ class IngestScheduler { * enqueued */ synchronized List getSourceContent() { - final Set contentSet = new HashSet(); + final Set contentSet = new HashSet<>(); - for (FileTask task : rootProcessTasks) { - contentSet.add(task.getDataSourceTask().getContent()); + for (FileTask task : rootDirectoryTasks) { + contentSet.add(task.getJob().getDataSource()); } - for (FileTask task : curDirProcessTasks) { - contentSet.add(task.getDataSourceTask().getContent()); + for (FileTask task : directoryTasks) { + contentSet.add(task.getJob().getDataSource()); } - for (FileTask task : curFileProcessTasks) { - contentSet.add(task.getDataSourceTask().getContent()); + for (FileTask task : fileTasks) { + contentSet.add(task.getJob().getDataSource()); } - return new ArrayList(contentSet); - } - - /** - * Determine if a module is in a pipeline in the queue. - * @param module - * @return true if it is in the queue. - */ - synchronized boolean hasModuleEnqueued(IngestModuleAbstractFile module) { - for (FileTask task : rootProcessTasks) { - List modules = task.getDataSourceTask().getModules(); - for (IngestModuleAbstractFile m : modules) { - if (m.getName().equals(module.getName())) { - return true; - } - } - } - - for (FileTask task : curDirProcessTasks) { - List modules = task.getDataSourceTask().getModules(); - for (IngestModuleAbstractFile m : modules) { - if (m.getName().equals(module.getName())) { - return true; - } - } - } - - for (FileTask task : curFileProcessTasks) { - List modules = task.getDataSourceTask().getModules(); - for (IngestModuleAbstractFile m : modules) { - if (m.getName().equals(module.getName())) { - return true; - } - } - } - - return false; + return new ArrayList<>(contentSet); } synchronized void empty() { - this.rootProcessTasks.clear(); - this.curDirProcessTasks.clear(); - this.curFileProcessTasks.clear(); + this.rootDirectoryTasks.clear(); + this.directoryTasks.clear(); + this.fileTasks.clear(); } /** @@ -588,7 +392,7 @@ class IngestScheduler { final AbstractFile aFile = processTask.file; //if it's unalloc file, skip if so scheduled - if (processTask.getDataSourceTask().isProcessUnalloc() == false + if (processTask.getJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS //unalloc files )) { return false; @@ -642,6 +446,70 @@ class IngestScheduler { return true; } + /** + * Task for a specific file to process. More specific than the + * higher-level DataSourceTask. + */ + static class FileTask { + private final AbstractFile file; + private final IngestJob task; + + public FileTask(AbstractFile file, IngestJob task) { + this.file = file; + this.task = task; + } + + public IngestJob getJob() { + return task; + } + + public AbstractFile getFile() { + return file; + } + + @Override + public String toString() { + try { + return "ProcessTask{" + "file=" + file.getId() + ": " + + file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}'; + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); + } + return "ProcessTask{" + "file=" + file.getId() + ": " + + file.getName() + '}'; + } + + /** + * two process tasks are equal when the file/dir and modules are the + * same this enables are not to queue up the same file/dir, modules + * tuples into the root dir set + * + * @param obj + * @return + */ + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final FileTask other = (FileTask) obj; + if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { + return false; + } + IngestJob thisTask = this.getJob(); + IngestJob otherTask = other.getJob(); + + if (thisTask != otherTask + && (thisTask == null || !thisTask.equals(otherTask))) { + return false; + } + return true; + } + } + /** * Root dir sorter */ @@ -888,61 +756,36 @@ class IngestScheduler { /** * DataSourceScheduler ingest scheduler */ - static class DataSourceScheduler implements Iterator> { + static class DataSourceScheduler implements Iterator { - private LinkedList> tasks; + private LinkedList tasks; DataSourceScheduler() { - tasks = new LinkedList>(); + tasks = new LinkedList<>(); } - synchronized void schedule(DataSourceTask task) { - - //skip if task contains no modules - if (task.getModules().isEmpty()) { - return; - } - + synchronized void schedule(IngestJob task) { try { - if (task.getContent().getParent() != null) { + if (task.getDataSource().getParent() != null) { //only accepting parent-less content objects (Image, parentless VirtualDirectory) - logger.log(Level.SEVERE, "Only parent-less Content (data sources) can be scheduled for DataSource ingest, skipping: " + task.getContent()); + logger.log(Level.SEVERE, "Only parent-less Content (data sources) can be scheduled for DataSource ingest, skipping: {0}", task.getDataSource()); return; } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error validating data source to be scheduled for DataSource ingest" + task.getContent(), e); + logger.log(Level.SEVERE, "Error validating data source to be scheduled for DataSource ingest" + task.getDataSource(), e); return; } - // see if we already have a task for this data source - DataSourceTask existTask = null; - for (DataSourceTask curTask : tasks) { - if (curTask.getContent().equals(task.getContent())) { - existTask = curTask; - break; - } - } - - // add these modules to the existing task for the data source - // @@@ BC: I'm not sure I like this and it will probably break a more formal pipeline structure - // @@@ TODO: Verify that if this is called mid-way during ingest that all of the already ingested files get scheduled with the new modules... - if (existTask != null) { - //merge modules for the data source task - existTask.addModules(task.getModules()); - } else { - //enqueue a new task - tasks.addLast(task); - } + tasks.addLast(task); } @Override - public synchronized DataSourceTask next() throws IllegalStateException { + public synchronized IngestJob next() throws IllegalStateException { if (!hasNext()) { - throw new IllegalStateException( - NbBundle.getMessage(this.getClass(), "IngestScheduler.DataSourceScheduler.exception.next.msg")); + throw new IllegalStateException("There is no data source tasks in the queue, check hasNext()"); } - final DataSourceTask ret = tasks.pollFirst(); + final IngestJob ret = tasks.pollFirst(); return ret; } @@ -953,8 +796,8 @@ class IngestScheduler { */ synchronized List getContents() { List contents = new ArrayList(); - for (DataSourceTask task : tasks) { - contents.add(task.getContent()); + for (IngestJob task : tasks) { + contents.add(task.getDataSource()); } return contents; } @@ -966,8 +809,7 @@ class IngestScheduler { @Override public void remove() { - throw new UnsupportedOperationException( - NbBundle.getMessage(this.getClass(), "IngestScheduler.DataSourceScheduler.exception.remove.msg")); + throw new UnsupportedOperationException("Removing of scheduled data source ingest tasks is not supported. "); } synchronized void empty() { @@ -981,12 +823,11 @@ class IngestScheduler { @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.DataSourceScheduler.toString.size", - getCount())); - for (DataSourceTask task : tasks) { + sb.append("DataSourceQueue, size: ").append(getCount()); + for (IngestJob task : tasks) { sb.append(task.toString()).append(" "); } return sb.toString(); } } -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java index e367cc3709..b269afb50f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,8 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - package org.sleuthkit.autopsy.ingest; import java.util.Map; @@ -28,27 +26,25 @@ import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.SleuthkitCase; - /** - * Singleton class that provides services for ingest modules. - * These exist to make it easier to write modules. Use the getDefault() - * method to get the singleton instance. + * Singleton class that provides services for ingest modules. These exist to + * make it easier to write modules. Use the getDefault() method to get the + * singleton instance. */ -public class IngestServices { - +public final class IngestServices { + + private static final Logger logger = Logger.getLogger(IngestServices.class.getName()); private IngestManager manager; - - private Logger logger = Logger.getLogger(IngestServices.class.getName()); - private static IngestServices instance; - + private IngestServices() { this.manager = IngestManager.getDefault(); } - + /** - * Get handle to singletone module services - * @return the services handle + * Get the ingest services. + * + * @return The ingest services singleton. */ public static synchronized IngestServices getDefault() { if (instance == null) { @@ -56,117 +52,90 @@ public class IngestServices { } return instance; } - + /** - * Get access to the current Case handle. - * Note: When storing the Case database handle as a member variable in a module, - * this method needs to be called within the module's init() method and the - * member variable needs to be updated at each init(), - * to ensure the correct Case handle is being used if the Case is changed. - * - * @return current Case + * Get the current Autopsy case. + * + * @return The current case. */ public Case getCurrentCase() { return Case.getCurrentCase(); } - - /** - * Get access to the current Case database handle. Like storing - * the Case handle, call this method and update member variables for each - * call to the module's init() method to ensure it is correct. - * - * @return current Case database + + /** + * Get the current SleuthKit case. The SleuthKit case is the case database. + * + * @return The current case database. */ public SleuthkitCase getCurrentSleuthkitCaseDb() { return Case.getCurrentCase().getSleuthkitCase(); } - - /** - * Get a logger to be used by the module to log messages to log files. - * @param module module to get the logger for - * @return logger object + + /** + * Get a logger that incorporates the display name of an ingest module in + * messages written to the Autopsy log files. + * + * @param moduleClassName The display name of the ingest module. + * @return The custom logger for the ingest module. */ - public Logger getLogger(IngestModuleAbstract module) { - return Logger.getLogger(module.getName()); + public Logger getLogger(String moduleDisplayName) { + return Logger.getLogger(moduleDisplayName); } - + /** - * Post ingest message to the inbox. This should be done for - * analysis messages. - * @param message ingest message to be posted by ingest module + * Post message to the ingest messages in box. + * + * @param message An ingest message */ public void postMessage(final IngestMessage message) { - manager.postMessage(message); + manager.postIngestMessage(message); } - + /** * Fire module event to notify registered module event listeners - * @param eventType the event type, defined in IngestManager.IngestManagerEvents + * + * @param eventType the event type, defined in + * IngestManager.IngestManagerEvents * @param moduleName the module name */ public void fireModuleEvent(String eventType, String moduleName) { IngestManager.fireModuleEvent(eventType, moduleName); } - /** - * Fire module data event to notify registered module data event listeners that there - * is new data of a given type from a module - * @param moduleDataEvent module data event, encapsulating blackboard artifact data + * Fire module data event to notify registered module data event listeners + * that there is new data of a given type from a module + * + * @param moduleDataEvent module data event, encapsulating blackboard + * artifact data */ public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { IngestManager.fireModuleDataEvent(moduleDataEvent); } - - - /** - * Fire module content event to notify registered module content event listeners - * that there is new content (from ZIP file contents, carving, etc.) - * @param moduleContentEvent module content event, encapsulating content changed + + /** + * Fire module content event to notify registered module content event + * listeners that there is new content (from ZIP file contents, carving, + * etc.) + * + * @param moduleContentEvent module content event, encapsulating content + * changed */ public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { IngestManager.fireModuleContentEvent(moduleContentEvent); } - + /** - * Schedule a new file for ingest with the same settings as the file - * being analyzed. This is used, for example, when opening an archive file. - * File needs to have already been added to the database. - * - * @param file file to be scheduled - * @param pipelineContext the ingest context for the file ingest pipeline - */ - public void scheduleFile(AbstractFile file, PipelineContext pipelineContext) { - logger.log(Level.INFO, "Scheduling file: " + file.getName()); - manager.scheduleFile(file, pipelineContext); - } - - - /** - * Get free disk space of a drive where ingest data are written to - * That drive is being monitored by IngestMonitor thread when ingest is running. - * + * Get free disk space of a drive where ingest data are written to That + * drive is being monitored by IngestMonitor thread when ingest is running. + * * @return amount of disk space, -1 if unknown */ public long getFreeDiskSpace() { return manager.getFreeDiskSpace(); - } + } - - - /** - * Facility for a file ingest module to check a return value from a previously run file ingest module - * that executed for the same file. - * The module return value can be used as a guideline to skip processing the file - * - * @param moduleName registered module name of the module to check the return value of - * @return the return value of the previously executed module for the currently processed file in the file ingest pipeline - */ - public IngestModuleAbstractFile.ProcessResult getAbstractFileModuleResult(String moduleName) { - return manager.getAbstractFileModuleResult(moduleName); - } - - /** + /** * Gets a specific name/value configuration setting for a module * @param moduleName moduleName identifier unique to that module * @param settingName setting name to retrieve diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Installer.java b/Core/src/org/sleuthkit/autopsy/ingest/Installer.java index 29e71af668..9ab47753f5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Installer.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/Installer.java @@ -51,7 +51,7 @@ public class Installer extends ModuleInstall { @Override public void run() { //at this point UI top component is present for sure, ensure manager has it - manager.initUI(); + manager.initIngestMessageInbox(); //force ingest inbox closed, even if previous state was open //IngestMessageTopComponent.findInstance().close(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/ModuleContentEvent.java b/Core/src/org/sleuthkit/autopsy/ingest/ModuleContentEvent.java index 0b2121a8a9..297236cd55 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/ModuleContentEvent.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/ModuleContentEvent.java @@ -43,7 +43,4 @@ public class ModuleContentEvent extends ChangeEvent { public String getModuleName() { return moduleName; } - - - } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/NoIngestModuleSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/NoIngestModuleSettings.java new file mode 100755 index 0000000000..1fb3b709f1 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/NoIngestModuleSettings.java @@ -0,0 +1,38 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +/** + * Implementation of the IngestModuleOptions interface for use by ingest modules + * that do not have per ingest job options. + */ +public final class NoIngestModuleSettings implements IngestModuleSettings { + + private final String setting = "None"; + + /** + * Gets the string used as an ingest options placeholder for serialization + * purposes. + * + * @return The string "None" + */ + String getSetting() { + return setting; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/PipelineContext.java b/Core/src/org/sleuthkit/autopsy/ingest/PipelineContext.java deleted file mode 100644 index 85e9fed5a1..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/PipelineContext.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2013 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import org.openide.util.NbBundle; - -import java.util.Objects; - -/** - * Stores information about a given pipeline, which is a series of modules. - * This is passed into modules for their reference. - * - * @param T type of the ingest associated with the context (file or data source Content) - * - */ -public class PipelineContext { - private final DataSourceTask task; - - PipelineContext(DataSourceTask task) { - this.task = task; - } - - /** - * Returns the currently scheduled task. - * @return - */ - DataSourceTask getDataSourceTask() { - return task; - } - - - @Override - public String toString() { - return NbBundle.getMessage(this.getClass(), "PipelineContext.toString.text", task); - } - - @Override - public int hashCode() { - int hash = 5; - hash = 53 * hash + Objects.hashCode(this.task); - return hash; - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - @SuppressWarnings("unchecked") - final PipelineContext other = (PipelineContext) obj; - - if (!Objects.equals(this.task, other.task)) { - return false; - } - - return true; - } -} diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index 5beff2bc7a..bdab0da9d9 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -963,7 +963,7 @@ import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM; StringBuilder summary = new StringBuilder(); boolean running = false; - if (IngestManager.getDefault().isIngestRunning() || IngestManager.getDefault().areModulesRunning()) { + if (IngestManager.getDefault().isIngestRunning() || IngestManager.getDefault().isIngestRunning()) { running = true; } diff --git a/ExifParser/nbproject/project.xml b/ExifParser/nbproject/project.xml index f9d480830f..88405985e9 100644 --- a/ExifParser/nbproject/project.xml +++ b/ExifParser/nbproject/project.xml @@ -11,7 +11,15 @@ - 8.25.1 + 8.29.3 + + + + org.openide.util.lookup + + + + 8.19.1 diff --git a/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java b/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java index 8f5ddccf5f..a0537977f1 100644 --- a/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java +++ b/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserFileIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,15 +33,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.logging.Level; - -import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.Version; -import org.sleuthkit.autopsy.ingest.PipelineContext; +import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; -import org.sleuthkit.autopsy.ingest.IngestModuleInit; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -57,59 +53,44 @@ import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM; * files. Ingests an image file and, if available, adds it's date, latitude, * longitude, altitude, device model, and device make to a blackboard artifact. */ -public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { +public final class ExifParserFileIngestModule extends IngestModuleAdapter implements FileIngestModule { - private IngestServices services; - final public static String MODULE_NAME = NbBundle.getMessage(ExifParserFileIngestModule.class, - "ExifParserFileIngestModule.moduleName.text"); - final public static String MODULE_VERSION = Version.getVersion(); private static final Logger logger = Logger.getLogger(ExifParserFileIngestModule.class.getName()); - private static ExifParserFileIngestModule defaultInstance = null; + private final IngestServices services = IngestServices.getDefault(); private int filesProcessed = 0; private boolean filesToFire = false; - //file ingest modules require a private constructor - //to ensure singleton instances - private ExifParserFileIngestModule() { - } - - //default instance used for module registration - public static synchronized ExifParserFileIngestModule getDefault() { - if (defaultInstance == null) { - defaultInstance = new ExifParserFileIngestModule(); - } - return defaultInstance; + ExifParserFileIngestModule() { } @Override - public IngestModuleAbstractFile.ProcessResult process(PipelineContext pipelineContext, AbstractFile content) { - + public ProcessResult process(AbstractFile content) { //skip unalloc if (content.getType().equals(TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { - return IngestModuleAbstractFile.ProcessResult.OK; + return ProcessResult.OK; } // skip known if (content.getKnown().equals(TskData.FileKnown.KNOWN)) { - return IngestModuleAbstractFile.ProcessResult.OK; + return ProcessResult.OK; } // update the tree every 1000 files if we have EXIF data that is not being being displayed filesProcessed++; if ((filesToFire) && (filesProcessed % 1000 == 0)) { - services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); + services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); filesToFire = false; } - + //skip unsupported if (!parsableFormat(content)) { - return IngestModuleAbstractFile.ProcessResult.OK; + return ProcessResult.OK; } return processFile(content); } - public IngestModuleAbstractFile.ProcessResult processFile(AbstractFile f) { + ProcessResult processFile(AbstractFile f) { InputStream in = null; BufferedInputStream bin = null; @@ -117,7 +98,7 @@ public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { in = new ReadContentInputStream(f); bin = new BufferedInputStream(in); - Collection attributes = new ArrayList(); + Collection attributes = new ArrayList<>(); Metadata metadata = ImageMetadataReader.readMetadata(bin, true); // Date @@ -125,7 +106,7 @@ public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { if (exifDir != null) { Date date = exifDir.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL); if (date != null) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED.getTypeID(), MODULE_NAME, date.getTime() / 1000)); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED.getTypeID(), ExifParserModuleFactory.getModuleName(), date.getTime() / 1000)); } } @@ -136,13 +117,13 @@ public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { if (loc != null) { double latitude = loc.getLatitude(); double longitude = loc.getLongitude(); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), MODULE_NAME, latitude)); - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID(), MODULE_NAME, longitude)); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), ExifParserModuleFactory.getModuleName(), latitude)); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID(), ExifParserModuleFactory.getModuleName(), longitude)); } - + Rational altitude = gpsDir.getRational(GpsDirectory.TAG_GPS_ALTITUDE); if (altitude != null) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE.getTypeID(), MODULE_NAME, altitude.doubleValue())); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE.getTypeID(), ExifParserModuleFactory.getModuleName(), altitude.doubleValue())); } } @@ -151,12 +132,12 @@ public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { if (devDir != null) { String model = devDir.getString(ExifIFD0Directory.TAG_MODEL); if (model != null && !model.isEmpty()) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_MODEL.getTypeID(), MODULE_NAME, model)); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_MODEL.getTypeID(), ExifParserModuleFactory.getModuleName(), model)); } - + String make = devDir.getString(ExifIFD0Directory.TAG_MAKE); if (make != null && !make.isEmpty()) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_MAKE.getTypeID(), MODULE_NAME, make)); + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_MAKE.getTypeID(), ExifParserModuleFactory.getModuleName(), make)); } } @@ -167,14 +148,16 @@ public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { filesToFire = true; } - return IngestModuleAbstractFile.ProcessResult.OK; - + return ProcessResult.OK; } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Failed to create blackboard artifact for exif metadata (" + ex.getLocalizedMessage() + ")."); + logger.log(Level.WARNING, "Failed to create blackboard artifact for exif metadata ({0}).", ex.getLocalizedMessage()); + return ProcessResult.ERROR; } catch (ImageProcessingException ex) { - logger.log(Level.WARNING, "Failed to process the image file: " + f.getParentPath() + "/" + f.getName() + "(" + ex.getLocalizedMessage() + ")"); + logger.log(Level.WARNING, "Failed to process the image file: {0}/{1}({2})", new Object[]{f.getParentPath(), f.getName(), ex.getLocalizedMessage()}); + return ProcessResult.ERROR; } catch (IOException ex) { logger.log(Level.WARNING, "IOException when parsing image file: " + f.getParentPath() + "/" + f.getName(), ex); + return ProcessResult.ERROR; } finally { try { if (in != null) { @@ -185,11 +168,9 @@ public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { } } catch (IOException ex) { logger.log(Level.WARNING, "Failed to close InputStream.", ex); + return ProcessResult.ERROR; } } - - // If we got here, there was an error - return IngestModuleAbstractFile.ProcessResult.ERROR; } /** @@ -205,44 +186,10 @@ public final class ExifParserFileIngestModule extends IngestModuleAbstractFile { } @Override - public void complete() { - logger.log(Level.INFO, "completed exif parsing " + this.toString()); + public void shutDown(boolean ingestJobCancelled) { if (filesToFire) { //send the final new data event - services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); + services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); } } - - @Override - public String getVersion() { - return MODULE_VERSION; - } - - @Override - public String getName() { - return NbBundle.getMessage(this.getClass(), "ExifParserFileIngestModule.getName.text"); - } - - @Override - public String getDescription() { - return NbBundle.getMessage(this.getClass(), "ExifParserFileIngestModule.getDesc.text"); - } - - @Override - public void init(IngestModuleInit initContext) throws IngestModuleException { - services = IngestServices.getDefault(); - logger.log(Level.INFO, "init() " + this.toString()); - - filesProcessed = 0; - filesToFire = false; - } - - @Override - public void stop() { - } - - @Override - public boolean hasBackgroundJobsRunning() { - return false; - } } \ No newline at end of file diff --git a/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserModuleFactory.java b/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserModuleFactory.java new file mode 100755 index 0000000000..639ed3bad6 --- /dev/null +++ b/ExifParser/src/org/sleuthkit/autopsy/exifparser/ExifParserModuleFactory.java @@ -0,0 +1,66 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.exifparser; + +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleFactory; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.openide.util.NbBundle; + +/** + * An factory that creates file ingest modules that parse EXIF meta data + * associated with media files (e.g., JPEG format files). + */ +@ServiceProvider(service = IngestModuleFactory.class) +public class ExifParserModuleFactory extends IngestModuleFactoryAdapter { + + @Override + public String getModuleDisplayName() { + return getModuleName(); + } + + static String getModuleName() { + return NbBundle.getMessage(ExifParserFileIngestModule.class, + "ExifParserFileIngestModule.moduleName.text"); + } + + @Override + public String getModuleDescription() { + return NbBundle.getMessage(ExifParserFileIngestModule.class, + "ExifParserFileIngestModule.getDesc.text"); + } + + @Override + public String getModuleVersionNumber() { + return Version.getVersion(); + } + + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleSettings ingestOptions) { + return new ExifParserFileIngestModule(); + } +} diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java index 728d8993cb..463178e25e 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/AddFileExtensionAction.java @@ -16,12 +16,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - package org.sleuthkit.autopsy.fileextmismatch; import org.openide.util.NbBundle; - import java.awt.event.ActionEvent; import java.util.ArrayList; import java.util.Arrays; @@ -30,36 +27,35 @@ import javax.swing.AbstractAction; import javax.swing.JOptionPane; /** - * Do the context menu action for adding a new filename extension to - * the mismatch list for the MIME type of the selected node. + * Do the context menu action for adding a new filename extension to the + * mismatch list for the MIME type of the selected node. */ -class AddFileExtensionAction extends AbstractAction { +class AddFileExtensionAction extends AbstractAction { + private String extStr; private String mimeTypeStr; - - public AddFileExtensionAction(String menuItemStr, String extStr, String mimeTypeStr) { + + public AddFileExtensionAction(String menuItemStr, String extStr, String mimeTypeStr) { super(menuItemStr); this.mimeTypeStr = mimeTypeStr; this.extStr = extStr; } - + @Override public void actionPerformed(ActionEvent event) { HashMap editableMap = FileExtMismatchXML.getDefault().load(); - ArrayList editedExtensions = new ArrayList<>(Arrays.asList(editableMap.get(mimeTypeStr))); + ArrayList editedExtensions = new ArrayList<>(Arrays.asList(editableMap.get(mimeTypeStr))); editedExtensions.add(extStr); - + // Old array will be replaced by new array for this key - editableMap.put(mimeTypeStr, editedExtensions.toArray(new String[0])); - - if (!FileExtMismatchXML.getDefault().save(editableMap)) { + editableMap.put(mimeTypeStr, editedExtensions.toArray(new String[0])); + + if (!FileExtMismatchXML.getDefault().save(editableMap)) { //error JOptionPane.showMessageDialog(null, - NbBundle.getMessage(this.getClass(), "AddFileExtensionAction.msgDlg.msg"), - NbBundle.getMessage(this.getClass(), "AddFileExtensionAction.msgDlg.title"), - JOptionPane.ERROR_MESSAGE); - } // else //in the future we might want to update the statusbar to give feedback to the user - - } - + NbBundle.getMessage(this.getClass(), "AddFileExtensionAction.msgDlg.msg"), + NbBundle.getMessage(this.getClass(), "AddFileExtensionAction.msgDlg.title"), + JOptionPane.ERROR_MESSAGE); + } // else //in the future we might want to update the statusbar to give feedback to the user + } } diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties index 9e235f5f04..be85efacb9 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle.properties @@ -1,22 +1,6 @@ OpenIDE-Module-Name=FileExtMismatch OptionsCategory_Name_FileExtMismatchOptions=File Extension Mismatch OptionsCategory_FileExtMismatch=File Extension Mismatch -FileExtMismatchConfigPanel.extHeaderLabel.text=Allowed Extensions: -FileExtMismatchConfigPanel.userExtTextField.text= -FileExtMismatchConfigPanel.addExtButton.text=Add Extension -FileExtMismatchConfigPanel.removeExtButton.text=Remove Selected Extension -FileExtMismatchConfigPanel.saveButton.text=Save Configuration -FileExtMismatchConfigPanel.jLabel1.text=File Types: -FileExtMismatchConfigPanel.removeTypeButton.text=Remove Selected Type -FileExtMismatchConfigPanel.addTypeButton.text=Add Type -FileExtMismatchConfigPanel.userTypeTextField.text= -FileExtMismatchConfigPanel.extErrorLabel.text=\ -FileExtMismatchConfigPanel.mimeErrLabel.text=\ -FileExtMismatchConfigPanel.mimeRemoveErrLabel.text=\ -FileExtMismatchConfigPanel.extRemoveErrLabel.text=\ -FileExtMismatchConfigPanel.saveMsgLabel.text=\ -FileExtMismatchSimpleConfigPanel.skipNoExtCheckBox.text=Skip files without extensions -FileExtMismatchSimpleConfigPanel.skipTextPlain.text=Skip text files AddFileExtensionAction.msgDlg.msg=Writing XML configuration file failed. AddFileExtensionAction.msgDlg.title=Add Mismatch Extension Error AddFileExtensionAction.extHeaderLbl.text=Allowed Extensions for @@ -50,3 +34,20 @@ FileExtMismatchIngestModule.complete.totalFiles=Total Files Processed FileExtMismatchIngestModule.complete.svcMsg.text=File Extension Mismatch Results FileExtMismatchOptionsPanelController.moduleErr=Module Error FileExtMismatchOptionsPanelController.moduleErr.msg=A module caused an error listening to FileExtMismatchOptionsPanelController updates. See log to determine which module. Some data could be incomplete. +FileExtMismatchModuleSettingsPanel.skipKnownFilesCheckbox.text=Skip known files (NSRL) +FileExtMismatchModuleSettingsPanel.skipTextPlain.text=Skip text files +FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text=Skip files without extensions +FileExtMismatchSettingsPanel.addTypeButton.text=Add Type +FileExtMismatchSettingsPanel.extErrorLabel.text=\ +FileExtMismatchSettingsPanel.extHeaderLabel.text=Allowed Extensions: +FileExtMismatchSettingsPanel.saveMsgLabel.text=\ +FileExtMismatchSettingsPanel.mimeRemoveErrLabel.text=\ +FileExtMismatchSettingsPanel.extRemoveErrLabel.text=\ +FileExtMismatchSettingsPanel.mimeErrLabel.text=\ +FileExtMismatchSettingsPanel.removeTypeButton.text=Remove Selected Type +FileExtMismatchSettingsPanel.saveButton.text=Save Configuration +FileExtMismatchSettingsPanel.jLabel1.text=File Types: +FileExtMismatchSettingsPanel.userExtTextField.text= +FileExtMismatchSettingsPanel.addExtButton.text=Add Extension +FileExtMismatchSettingsPanel.removeExtButton.text=Remove Selected Extension +FileExtMismatchSettingsPanel.userTypeTextField.text= diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle_ja.properties b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle_ja.properties index d172dfb1d8..5ab739bd53 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle_ja.properties +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/Bundle_ja.properties @@ -1,45 +1,46 @@ -OpenIDE-Module-Name=\u30D5\u30A1\u30A4\u30EB\u62E1\u5F35\u5B50\u4E0D\u4E00\u81F4 -OptionsCategory_Name_FileExtMismatchOptions=\u30D5\u30A1\u30A4\u30EB\u62E1\u5F35\u5B50\u4E0D\u4E00\u81F4 -OptionsCategory_FileExtMismatch=\u30D5\u30A1\u30A4\u30EB\u62E1\u5F35\u5B50\u4E0D\u4E00\u81F4 -FileExtMismatchConfigPanel.extHeaderLabel.text=\u8A31\u53EF\u3055\u308C\u305F\u62E1\u5F35\u5B50\uFF1A -FileExtMismatchConfigPanel.addExtButton.text=\u62E1\u5F35\u5B50\u3092\u8FFD\u52A0 -FileExtMismatchConfigPanel.removeExtButton.text=\u9078\u629E\u3057\u305F\u62E1\u5F35\u5B50\u3092\u524A\u9664 -FileExtMismatchConfigPanel.saveButton.text=\u8A2D\u5B9A\u3092\u4FDD\u5B58 -FileExtMismatchConfigPanel.jLabel1.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\uFF1A -FileExtMismatchConfigPanel.removeTypeButton.text=\u9078\u629E\u3057\u305F\u30BF\u30A4\u30D7\u3092\u524A\u9664 -FileExtMismatchConfigPanel.addTypeButton.text=\u30BF\u30A4\u30D7\u3092\u8FFD\u52A0 -FileExtMismatchSimpleConfigPanel.skipNoExtCheckBox.text=\u62E1\u5F35\u5B50\u306E\u7121\u3044\u30D5\u30A1\u30A4\u30EB\u306F\u30B9\u30AD\u30C3\u30D7 -FileExtMismatchSimpleConfigPanel.skipTextPlain.text=\u30C6\u30AD\u30B9\u30C8\u30D5\u30A1\u30A4\u30EB\u306F\u30B9\u30AD\u30C3\u30D7 -AddFileExtensionAction.msgDlg.msg=XML\u8A2D\u5B9A\u30D5\u30A1\u30A4\u30EB\u3092\u66F8\u304F\u306E\u3092\u5931\u6557\u3057\u307E\u3057\u305F\u3002 -AddFileExtensionAction.msgDlg.title=\u4E0D\u4E00\u81F4\u62E1\u5F35\u5B50\u306E\u8FFD\u52A0\u30A8\u30E9\u30FC -FileExtMismatchConfigPanel.name.text=\u30A2\u30C9\u30D0\u30F3\u30B9\u30D5\u30A1\u30A4\u30EB\u62E1\u5F35\u5B50\u4E0D\u4E00\u81F4\u8A2D\u5B9A -FileExtMismatchConfigPanel.addExtButton.errLabel.empty=\u62E1\u5F35\u5B50\u30C6\u30AD\u30B9\u30C8\u304C\u7A7A\u767D\u3067\u3059\uFF01 -FileExtMismatchConfigPanel.addExtButton.errLabel.noMimeType=MIME\u30BF\u30A4\u30D7\u304C\u9078\u629E\u3055\u308C\u3066\u3044\u307E\u305B\u3093\uFF01 -FileExtMismatchConfigPanel.addExtButton.errLabel.extExists=\u62E1\u5F35\u5B50\u306F\u3059\u3067\u306B\u5B58\u5728\u3057\u307E\u3059\uFF01 -FileExtMismatchConfigPanel.addExtButton.errLabel.extAdded=\u62E1\u5F35\u5B50{0}\u306F\u8FFD\u52A0\u3055\u308C\u307E\u3057\u305F\u3002 -FileExtMismatchConfigPanel.addTypeButton.empty=MIME\u30BF\u30A4\u30D7\u30C6\u30AD\u30B9\u30C8\u304C\u7A7A\u767D\u3067\u3059\uFF01 -FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotSupported=MIME\u30BF\u30A4\u30D7\u306F\u30B5\u30DD\u30FC\u30C8\u3055\u308C\u3066\u3044\u307E\u305B\u3093\uFF01 -FileExtMismatchConfigPanel.addTypeButton.mimeTypeExists=MIME\u30BF\u30A4\u30D7\u306F\u3059\u3067\u306B\u5B58\u5728\u3057\u307E\u3059\uFF01 -FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotDetectable=MIME\u30BF\u30A4\u30D7\u306F\u3053\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u3067\u306F\u691C\u51FA\u3067\u304D\u307E\u305B\u3093\u3002 -FileExtMismatchConfigPanel.addTypeButton.mimeTypeAdded=MIME\u30BF\u30A4\u30D7{0}\u306F\u8FFD\u52A0\u3055\u308C\u307E\u3057\u305F\u3002 -FileExtMismatchConfigPanel.removeTypeButton.noneSelected=MIME\u30BF\u30A4\u30D7\u304C\u9078\u629E\u3055\u308C\u3066\u3044\u307E\u305B\u3093\uFF01 -FileExtMismatchConfigPanel.remoteTypeButton.deleted=MIME\u30BF\u30A4\u30D7{0}\u306F\u524A\u9664\u3055\u308C\u307E\u3057\u305F\u3002 -FileExtMismatchConfigPanel.removeExtButton.noneSelected=\u62E1\u5F35\u5B50\u304C\u9078\u629E\u3055\u308C\u3066\u3044\u307E\u305B\u3093\uFF01 -FileExtMismatchConfigPanel.removeExtButton.noMimeTypeSelected=MIME\u30BF\u30A4\u30D7\u304C\u9078\u629E\u3055\u308C\u3066\u3044\u307E\u305B\u3093\uFF01 -FileExtMismatchConfigPanel.removeExtButton.deleted=\u62E1\u5F35\u5B50{0}\u306F\u524A\u9664\u3055\u308C\u307E\u3057\u305F\u3002 -FileExtMismatchConfigPanel.store.msg=\u4FDD\u5B58\u3055\u308C\u307E\u3057\u305F\u3002 -FileExtMismatchConfigPanel.store.msgDlg.msg=XML\u8A2D\u5B9A\u30D5\u30A1\u30A4\u30EB\u3092\u66F8\u304F\u306E\u3092\u5931\u6557\u3057\u307E\u3057\u305F\u3002 -FileExtMismatchConfigPanel.save.msgDlg.title=\u4FDD\u5B58\u30A8\u30E9\u30FC -FileExtMismatchConfigPanel.ok.confDlg.msg=\u8A2D\u5B9A\u5909\u66F4\u3092\u4FDD\u5B58\u3057\u307E\u3059\u304B\uFF1F -FileExtMismatchConfigPanel.confDlg.title=\u4FDD\u5B58\u3055\u308C\u3066\u3044\u306A\u3044\u5909\u66F4 -FileExtMismatchConfigPanel.mimeTableModel.colName=MIME\u30BF\u30A4\u30D7 -FileExtMismatchConfigPanel.extTableModel.colName=\u62E1\u5F35\u5B50 -FileExtMismatchContextMenuActionsProvider.menuItemStr=\u62E1\u5F35\u5B50{0}\u3092MIME\u30BF\u30A4\u30D7{1}\u306E\u4E00\u81F4\u3068\u3057\u3066\u8FFD\u52A0 -FileExtMismatchIngestModule.moduleName=\u62E1\u5F35\u5B50\u4E0D\u4E00\u81F4\u30C7\u30A3\u30C6\u30AF\u30BF\u30FC -FileExtMismatchIngestModule.moduleDesc.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306B\u57FA\u3065\u3044\u3066\u3001\u6A19\u6E96\u7684\u3067\u306F\u306A\u3044\u62E1\u5F35\u5B50\u3092\u6301\u3064\u30D5\u30A1\u30A4\u30EB\u3092\u30D5\u30E9\u30B0\u4ED8\u3051\u3057\u307E\u3059\u3002d -FileExtMismatchIngestModule.complete.totalProcTime=\u5408\u8A08\u51E6\u7406\u6642\u9593 -FileExtMismatchIngestModule.complete.totalFiles=\u5408\u8A08\u51E6\u7406\u30D5\u30A1\u30A4\u30EB\u6570 -FileExtMismatchIngestModule.complete.svcMsg.text=\u30D5\u30A1\u30A4\u30EB\u62E1\u5F35\u5B50\u4E0D\u4E00\u81F4\u7D50\u679C -FileExtMismatchOptionsPanelController.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC -FileExtMismatchOptionsPanelController.moduleErr.msg=FileExtMismatchOptionsPanelController\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u306E\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 -AddFileExtensionAction.extHeaderLbl.text=\u4E0B\u8A18\u7528\u306B\u8A31\u53EF\u3055\u308C\u305F\u62E1\u5F35\u5B50 \ No newline at end of file +OpenIDE-Module-Name=\u30d5\u30a1\u30a4\u30eb\u62e1\u5f35\u5b50\u4e0d\u4e00\u81f4 +OptionsCategory_Name_FileExtMismatchOptions=\u30d5\u30a1\u30a4\u30eb\u62e1\u5f35\u5b50\u4e0d\u4e00\u81f4 +OptionsCategory_FileExtMismatch=\u30d5\u30a1\u30a4\u30eb\u62e1\u5f35\u5b50\u4e0d\u4e00\u81f4 +AddFileExtensionAction.msgDlg.msg=XML\u8a2d\u5b9a\u30d5\u30a1\u30a4\u30eb\u3092\u66f8\u304f\u306e\u3092\u5931\u6557\u3057\u307e\u3057\u305f\u3002 +AddFileExtensionAction.msgDlg.title=\u4e0d\u4e00\u81f4\u62e1\u5f35\u5b50\u306e\u8ffd\u52a0\u30a8\u30e9\u30fc +FileExtMismatchConfigPanel.name.text=\u30a2\u30c9\u30d0\u30f3\u30b9\u30d5\u30a1\u30a4\u30eb\u62e1\u5f35\u5b50\u4e0d\u4e00\u81f4\u8a2d\u5b9a +FileExtMismatchConfigPanel.addExtButton.errLabel.empty=\u62e1\u5f35\u5b50\u30c6\u30ad\u30b9\u30c8\u304c\u7a7a\u767d\u3067\u3059\uff01 +FileExtMismatchConfigPanel.addExtButton.errLabel.noMimeType=MIME\u30bf\u30a4\u30d7\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093\uff01 +FileExtMismatchConfigPanel.addExtButton.errLabel.extExists=\u62e1\u5f35\u5b50\u306f\u3059\u3067\u306b\u5b58\u5728\u3057\u307e\u3059\uff01 +FileExtMismatchConfigPanel.addExtButton.errLabel.extAdded=\u62e1\u5f35\u5b50{0}\u306f\u8ffd\u52a0\u3055\u308c\u307e\u3057\u305f\u3002 +FileExtMismatchConfigPanel.addTypeButton.empty=MIME\u30bf\u30a4\u30d7\u30c6\u30ad\u30b9\u30c8\u304c\u7a7a\u767d\u3067\u3059\uff01 +FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotSupported=MIME\u30bf\u30a4\u30d7\u306f\u30b5\u30dd\u30fc\u30c8\u3055\u308c\u3066\u3044\u307e\u305b\u3093\uff01 +FileExtMismatchConfigPanel.addTypeButton.mimeTypeExists=MIME\u30bf\u30a4\u30d7\u306f\u3059\u3067\u306b\u5b58\u5728\u3057\u307e\u3059\uff01 +FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotDetectable=MIME\u30bf\u30a4\u30d7\u306f\u3053\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u3067\u306f\u691c\u51fa\u3067\u304d\u307e\u305b\u3093\u3002 +FileExtMismatchConfigPanel.addTypeButton.mimeTypeAdded=MIME\u30bf\u30a4\u30d7{0}\u306f\u8ffd\u52a0\u3055\u308c\u307e\u3057\u305f\u3002 +FileExtMismatchConfigPanel.removeTypeButton.noneSelected=MIME\u30bf\u30a4\u30d7\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093\uff01 +FileExtMismatchConfigPanel.remoteTypeButton.deleted=MIME\u30bf\u30a4\u30d7{0}\u306f\u524a\u9664\u3055\u308c\u307e\u3057\u305f\u3002 +FileExtMismatchConfigPanel.removeExtButton.noneSelected=\u62e1\u5f35\u5b50\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093\uff01 +FileExtMismatchConfigPanel.removeExtButton.noMimeTypeSelected=MIME\u30bf\u30a4\u30d7\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093\uff01 +FileExtMismatchConfigPanel.removeExtButton.deleted=\u62e1\u5f35\u5b50{0}\u306f\u524a\u9664\u3055\u308c\u307e\u3057\u305f\u3002 +FileExtMismatchConfigPanel.store.msg=\u4fdd\u5b58\u3055\u308c\u307e\u3057\u305f\u3002 +FileExtMismatchConfigPanel.store.msgDlg.msg=XML\u8a2d\u5b9a\u30d5\u30a1\u30a4\u30eb\u3092\u66f8\u304f\u306e\u3092\u5931\u6557\u3057\u307e\u3057\u305f\u3002 +FileExtMismatchConfigPanel.save.msgDlg.title=\u4fdd\u5b58\u30a8\u30e9\u30fc +FileExtMismatchConfigPanel.ok.confDlg.msg=\u8a2d\u5b9a\u5909\u66f4\u3092\u4fdd\u5b58\u3057\u307e\u3059\u304b\uff1f +FileExtMismatchConfigPanel.confDlg.title=\u4fdd\u5b58\u3055\u308c\u3066\u3044\u306a\u3044\u5909\u66f4 +FileExtMismatchConfigPanel.mimeTableModel.colName=MIME\u30bf\u30a4\u30d7 +FileExtMismatchConfigPanel.extTableModel.colName=\u62e1\u5f35\u5b50 +FileExtMismatchContextMenuActionsProvider.menuItemStr=\u62e1\u5f35\u5b50{0}\u3092MIME\u30bf\u30a4\u30d7{1}\u306e\u4e00\u81f4\u3068\u3057\u3066\u8ffd\u52a0 +FileExtMismatchIngestModule.moduleName=\u62e1\u5f35\u5b50\u4e0d\u4e00\u81f4\u30c7\u30a3\u30c6\u30af\u30bf\u30fc +FileExtMismatchIngestModule.moduleDesc.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306b\u57fa\u3065\u3044\u3066\u3001\u6a19\u6e96\u7684\u3067\u306f\u306a\u3044\u62e1\u5f35\u5b50\u3092\u6301\u3064\u30d5\u30a1\u30a4\u30eb\u3092\u30d5\u30e9\u30b0\u4ed8\u3051\u3057\u307e\u3059\u3002d +FileExtMismatchIngestModule.complete.totalProcTime=\u5408\u8a08\u51e6\u7406\u6642\u9593 +FileExtMismatchIngestModule.complete.totalFiles=\u5408\u8a08\u51e6\u7406\u30d5\u30a1\u30a4\u30eb\u6570 +FileExtMismatchIngestModule.complete.svcMsg.text=\u30d5\u30a1\u30a4\u30eb\u62e1\u5f35\u5b50\u4e0d\u4e00\u81f4\u7d50\u679c +FileExtMismatchOptionsPanelController.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc +FileExtMismatchOptionsPanelController.moduleErr.msg=FileExtMismatchOptionsPanelController\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u306e\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 +AddFileExtensionAction.extHeaderLbl.text=\u4e0b\u8a18\u7528\u306b\u8a31\u53ef\u3055\u308c\u305f\u62e1\u5f35\u5b50 +FileExtMismatchModuleSettingsPanel.skipTextPlain.text=\u30c6\u30ad\u30b9\u30c8\u30d5\u30a1\u30a4\u30eb\u306f\u30b9\u30ad\u30c3\u30d7 + +FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text=\u62e1\u5f35\u5b50\u306e\u7121\u3044\u30d5\u30a1\u30a4\u30eb\u306f\u30b9\u30ad\u30c3\u30d7 +FileExtMismatchSettingsPanel.addTypeButton.text=\u30bf\u30a4\u30d7\u3092\u8ffd\u52a0 +FileExtMismatchSettingsPanel.extHeaderLabel.text=\u8a31\u53ef\u3055\u308c\u305f\u62e1\u5f35\u5b50\uff1a +FileExtMismatchSettingsPanel.removeTypeButton.text=\u9078\u629e\u3057\u305f\u30bf\u30a4\u30d7\u3092\u524a\u9664 +FileExtMismatchSettingsPanel.saveButton.text=\u8a2d\u5b9a\u3092\u4fdd\u5b58 +FileExtMismatchSettingsPanel.jLabel1.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\uff1a +FileExtMismatchSettingsPanel.addExtButton.text=\u62e1\u5f35\u5b50\u3092\u8ffd\u52a0 +FileExtMismatchSettingsPanel.removeExtButton.text=\u9078\u629e\u3057\u305f\u62e1\u5f35\u5b50\u3092\u524a\u9664 diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java index a6e422472f..2409baee49 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchContextMenuActionsProvider.java @@ -27,14 +27,13 @@ import java.util.HashMap; import java.util.List; import java.util.logging.Level; import javax.swing.Action; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.Utilities; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.corecomponentinterfaces.ContextMenuActionsProvider; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.ingest.IngestConfigurator; +import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -51,8 +50,7 @@ public class FileExtMismatchContextMenuActionsProvider implements ContextMenuAct ArrayList actions = new ArrayList<>(); // Ignore if file ingest is in progress. - IngestConfigurator ingestConfigurator = Lookup.getDefault().lookup(IngestConfigurator.class); - if (ingestConfigurator != null && !ingestConfigurator.isIngestRunning()) { + if (!IngestManager.getDefault().isIngestRunning()) { final Collection selectedArts = Utilities.actionsGlobalContext().lookupAll(BlackboardArtifact.class); @@ -127,4 +125,4 @@ public class FileExtMismatchContextMenuActionsProvider implements ContextMenuAct return actions; } -} +} \ No newline at end of file diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleFactory.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleFactory.java new file mode 100755 index 0000000000..ee9b36684a --- /dev/null +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleFactory.java @@ -0,0 +1,104 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.fileextmismatch; + +import org.openide.util.NbBundle; +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleFactory; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleSettingsPanel; +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel; + +/** + * An factory that creates file ingest modules that detect mismatches between + * the types of files and their extensions. + */ +@ServiceProvider(service = IngestModuleFactory.class) +public class FileExtMismatchDetectorModuleFactory extends IngestModuleFactoryAdapter { + + static String getModuleName() { + return NbBundle.getMessage(FileExtMismatchIngestModule.class, + "FileExtMismatchIngestModule.moduleName"); + } + + @Override + public String getModuleDisplayName() { + return getModuleName(); + } + + @Override + public String getModuleDescription() { + return NbBundle.getMessage(FileExtMismatchIngestModule.class, + "FileExtMismatchIngestModule.moduleDesc.text"); + } + + @Override + public String getModuleVersionNumber() { + return Version.getVersion(); + } + + @Override + public IngestModuleSettings getDefaultModuleSettings() { + return new FileExtMismatchDetectorModuleSettings(); + } + + @Override + public boolean hasModuleSettingsPanel() { + return true; + } + + @Override + public IngestModuleSettingsPanel getModuleSettingsPanel(IngestModuleSettings settings) { + assert settings instanceof FileExtMismatchDetectorModuleSettings; + if (!(settings instanceof FileExtMismatchDetectorModuleSettings)) { + throw new IllegalArgumentException("Expected settings argument to be instanceof FileExtMismatchDetectorModuleSettings"); + } + FileExtMismatchModuleSettingsPanel settingsPanel = new FileExtMismatchModuleSettingsPanel((FileExtMismatchDetectorModuleSettings) settings); + return settingsPanel; + } + + @Override + public boolean hasGlobalSettingsPanel() { + return true; + } + + @Override + public IngestModuleGlobalSetttingsPanel getGlobalSettingsPanel() { + FileExtMismatchSettingsPanel globalOptionsPanel = new FileExtMismatchSettingsPanel(); + globalOptionsPanel.load(); + return globalOptionsPanel; + } + + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleSettings settings) { + assert settings instanceof FileExtMismatchDetectorModuleSettings; + if (!(settings instanceof FileExtMismatchDetectorModuleSettings)) { + throw new IllegalArgumentException("Expected settings argument to be instanceof FileExtMismatchDetectorModuleSettings"); + } + return new FileExtMismatchIngestModule((FileExtMismatchDetectorModuleSettings) settings); + } +} diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleSettings.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleSettings.java new file mode 100755 index 0000000000..eda28ab6ff --- /dev/null +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchDetectorModuleSettings.java @@ -0,0 +1,64 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.fileextmismatch; + +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; + +/** + * Ingest options for the file extension mismatch detector ingest module. + */ +final class FileExtMismatchDetectorModuleSettings implements IngestModuleSettings { + + private boolean skipKnownFiles = false; + private boolean skipFilesWithNoExtension = true; + private boolean skipFilesWithTextPlainMimeType = false; + + FileExtMismatchDetectorModuleSettings() { + } + + FileExtMismatchDetectorModuleSettings(boolean skipKnownFiles, boolean skipFilesWithNoExtension, boolean skipFilesWithTextPlainMimeType) { + this.skipKnownFiles = skipKnownFiles; + this.skipFilesWithNoExtension = skipFilesWithNoExtension; + this.skipFilesWithTextPlainMimeType = skipFilesWithTextPlainMimeType; + } + + void setSkipKnownFiles(boolean enabled) { + skipKnownFiles = enabled; + } + + boolean skipKnownFiles() { + return skipKnownFiles; + } + + void setSkipFilesWithNoExtension(boolean enabled) { + skipFilesWithNoExtension = enabled; + } + + boolean skipFilesWithNoExtension() { + return skipFilesWithNoExtension; + } + + void setSkipFilesWithTextPlainMimeType(boolean enabled) { + skipFilesWithTextPlainMimeType = enabled; + } + + boolean skipFilesWithTextPlainMimeType() { + return skipFilesWithTextPlainMimeType; + } +} diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java index 61c0d3ff52..42cd7e699a 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 - 2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,8 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - package org.sleuthkit.autopsy.fileextmismatch; import java.util.ArrayList; @@ -26,16 +24,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.logging.Level; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; -import org.sleuthkit.autopsy.ingest.IngestModuleInit; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.ingest.PipelineContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -48,86 +44,57 @@ import org.sleuthkit.datamodel.TskException; /** * Flags mismatched filename extensions based on file signature. */ -public class FileExtMismatchIngestModule extends org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile { - private static FileExtMismatchIngestModule defaultInstance = null; - private static final Logger logger = Logger.getLogger(FileExtMismatchIngestModule.class.getName()); - public static final String MODULE_NAME = NbBundle.getMessage(FileExtMismatchIngestModule.class, - "FileExtMismatchIngestModule.moduleName"); - public static final String MODULE_DESCRIPTION = NbBundle.getMessage(FileExtMismatchIngestModule.class, - "FileExtMismatchIngestModule.moduleDesc.text"); - public static final String MODULE_VERSION = Version.getVersion(); +public class FileExtMismatchIngestModule extends IngestModuleAdapter implements FileIngestModule { - private static long processTime = 0; - private static int messageId = 0; - private static long numFiles = 0; - // note: because of current design, these values must be in sync with default GUI values - // they only get updated when the user changes from the default UI values - private static boolean skipNoExt = true; - private static boolean skipTextPlain = true; - - private FileExtMismatchSimpleConfigPanel simpleConfigPanel; - private FileExtMismatchConfigPanel advancedConfigPanel; - private IngestServices services; + private static final Logger logger = Logger.getLogger(FileExtMismatchIngestModule.class.getName()); + private static int messageId = 0; // RJCTODO: This is not thread safe + private final IngestServices services = IngestServices.getDefault(); + private final FileExtMismatchDetectorModuleSettings settings; private HashMap SigTypeToExtMap = new HashMap<>(); - - - // Private to ensure Singleton status - private FileExtMismatchIngestModule() { + private long processTime = 0; + private long numFiles = 0; - } - - // File-level ingest modules are currently singleton -- this is required - public static synchronized FileExtMismatchIngestModule getDefault() { - //defaultInstance is a private static class variable - if (defaultInstance == null) { - defaultInstance = new FileExtMismatchIngestModule(); - } - return defaultInstance; + FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) { + this.settings = settings; } - @Override - public void init(IngestModuleInit initContext) throws IngestModuleException { - services = IngestServices.getDefault(); - - // Load mapping + public void startUp(IngestJobContext context) throws IngestModuleException { FileExtMismatchXML xmlLoader = FileExtMismatchXML.getDefault(); SigTypeToExtMap = xmlLoader.load(); - } - + @Override - public ProcessResult process(PipelineContext pipelineContext, AbstractFile abstractFile) { + public ProcessResult process(AbstractFile abstractFile) { // skip non-files - if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) || - (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { - return ProcessResult.OK; - } - - // deleted files often have content that was not theirs and therefor causes mismatch - if ((abstractFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.UNALLOC)) || - (abstractFile.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC))) { + if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { return ProcessResult.OK; } - if (abstractFile.getKnown() == FileKnown.KNOWN) { + // deleted files often have content that was not theirs and therefor causes mismatch + if ((abstractFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.UNALLOC)) + || (abstractFile.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC))) { return ProcessResult.OK; } - - try - { + + if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) { + return ProcessResult.OK; + } + + try { long startTime = System.currentTimeMillis(); - + boolean mismatchDetected = compareSigTypeToExt(abstractFile); - + processTime += (System.currentTimeMillis() - startTime); numFiles++; - + if (mismatchDetected) { // add artifact BlackboardArtifact bart = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED); - services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED, Collections.singletonList(bart))); + services.fireModuleDataEvent(new ModuleDataEvent(FileExtMismatchDetectorModuleFactory.getModuleName(), ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED, Collections.singletonList(bart))); } return ProcessResult.OK; } catch (TskException ex) { @@ -135,18 +102,19 @@ public class FileExtMismatchIngestModule extends org.sleuthkit.autopsy.ingest.In return ProcessResult.ERROR; } } - + /** * Compare file type for file and extension. + * * @param abstractFile - * @return false if the two match. True if there is a mismatch. + * @return false if the two match. True if there is a mismatch. */ private boolean compareSigTypeToExt(AbstractFile abstractFile) { try { String currActualExt = abstractFile.getNameExtension(); // If we are skipping names with no extension - if (skipNoExt && currActualExt.isEmpty()) { + if (settings.skipFilesWithNoExtension() && currActualExt.isEmpty()) { return false; } @@ -155,7 +123,7 @@ public class FileExtMismatchIngestModule extends org.sleuthkit.autopsy.ingest.In ArrayList attributes = abstractFile.getGenInfoAttributes(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG); for (BlackboardAttribute attribute : attributes) { String currActualSigType = attribute.getValueString(); - if (skipTextPlain) { + if (settings.skipFilesWithTextPlainMimeType()) { if (!currActualExt.isEmpty() && currActualSigType.equals("text/plain")) { return false; } @@ -183,97 +151,22 @@ public class FileExtMismatchIngestModule extends org.sleuthkit.autopsy.ingest.In return false; } - + @Override - public void complete() { + public void shutDown(boolean ingestJobCancelled) { StringBuilder detailsSb = new StringBuilder(); - //details detailsSb.append(""); - - detailsSb.append(""); - + detailsSb.append(""); detailsSb.append("\n"); + .append("\n"); detailsSb.append("\n"); + .append("\n"); detailsSb.append("
"+MODULE_DESCRIPTION+"
").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("
").append( NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime")) - .append("").append(processTime).append("
").append(processTime).append("
").append( NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles")) - .append("").append(numFiles).append("
").append(numFiles).append("
"); - - services.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, this, - NbBundle.getMessage(this.getClass(), - "FileExtMismatchIngestModule.complete.svcMsg.text"), - detailsSb.toString())); - } - - @Override - public void stop() { - //do nothing - } - - @Override - public String getName() { - return MODULE_NAME; - } - - @Override - public String getDescription() { - return MODULE_DESCRIPTION; - } - - @Override - public String getVersion() { - return MODULE_VERSION; - } - - @Override - public boolean hasSimpleConfiguration() { - return true; - } - - @Override - public boolean hasAdvancedConfiguration() { - return true; - } - - @Override - public javax.swing.JPanel getSimpleConfiguration(String context) { - if (simpleConfigPanel == null) { - simpleConfigPanel = new FileExtMismatchSimpleConfigPanel(); - } - - return simpleConfigPanel; - } - - @Override - public javax.swing.JPanel getAdvancedConfiguration(String context) { - getPanel().load(); - return getPanel(); - } - - private FileExtMismatchConfigPanel getPanel() { - if (advancedConfigPanel == null) { - advancedConfigPanel = new FileExtMismatchConfigPanel(); - } - return advancedConfigPanel; - } - - @Override - public void saveAdvancedConfiguration() { - getPanel().store(); - } - - @Override - public boolean hasBackgroundJobsRunning() { - // we're single threaded... - return false; - } - - protected static void setSkipNoExt(boolean flag) { - skipNoExt = flag; - } - protected static void setSkipTextPlain(boolean flag) { - skipTextPlain = flag; + services.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileExtMismatchIngestModule.complete.svcMsg.text"), + detailsSb.toString())); } } - diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSimpleConfigPanel.form b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.form similarity index 70% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSimpleConfigPanel.form rename to FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.form index ac705a68f9..a2e7553626 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSimpleConfigPanel.form +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.form @@ -21,6 +21,7 @@ + @@ -32,7 +33,9 @@ - + + + @@ -42,7 +45,7 @@ - + @@ -52,7 +55,7 @@ - + @@ -62,5 +65,15 @@ + + + + + + + + + + diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSimpleConfigPanel.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.java similarity index 56% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSimpleConfigPanel.java rename to FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.java index 13068908ba..8bdc1a5f7c 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSimpleConfigPanel.java +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchModuleSettingsPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 - 2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,26 +18,38 @@ */ package org.sleuthkit.autopsy.fileextmismatch; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleSettingsPanel; /** - * Instances of this class provide a simplified UI for managing the hash sets configuration. + * UI component used to set ingest job options for file extension mismatch + * detector ingest modules. */ -class FileExtMismatchSimpleConfigPanel extends javax.swing.JPanel { +final class FileExtMismatchModuleSettingsPanel extends IngestModuleSettingsPanel { - public FileExtMismatchSimpleConfigPanel() { + private final FileExtMismatchDetectorModuleSettings settings; + + FileExtMismatchModuleSettingsPanel(FileExtMismatchDetectorModuleSettings settings) { + this.settings = settings; initComponents(); customizeComponents(); } - - private void customizeComponents() { + private void customizeComponents() { + skipNoExtCheckBox.setSelected(settings.skipFilesWithNoExtension()); + skipTextPlain.setSelected(settings.skipFilesWithTextPlainMimeType()); + skipKnownFilesCheckbox.setSelected(settings.skipKnownFiles()); } - - - /** This method is called from within the constructor to - * initialize the form. - * WARNING: Do NOT modify this code. The content of this method is - * always regenerated by the Form Editor. + + @Override + public IngestModuleSettings getSettings() { + return settings; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. */ @SuppressWarnings("unchecked") // //GEN-BEGIN:initComponents @@ -45,16 +57,17 @@ class FileExtMismatchSimpleConfigPanel extends javax.swing.JPanel { skipNoExtCheckBox = new javax.swing.JCheckBox(); skipTextPlain = new javax.swing.JCheckBox(); + skipKnownFilesCheckbox = new javax.swing.JCheckBox(); skipNoExtCheckBox.setSelected(true); - skipNoExtCheckBox.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSimpleConfigPanel.class, "FileExtMismatchSimpleConfigPanel.skipNoExtCheckBox.text")); // NOI18N + skipNoExtCheckBox.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchModuleSettingsPanel.class, "FileExtMismatchModuleSettingsPanel.skipNoExtCheckBox.text")); // NOI18N skipNoExtCheckBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { skipNoExtCheckBoxActionPerformed(evt); } }); - skipTextPlain.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSimpleConfigPanel.class, "FileExtMismatchSimpleConfigPanel.skipTextPlain.text")); // NOI18N + skipTextPlain.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchModuleSettingsPanel.class, "FileExtMismatchModuleSettingsPanel.skipTextPlain.text")); // NOI18N skipTextPlain.setSelected(true); skipTextPlain.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -62,6 +75,13 @@ class FileExtMismatchSimpleConfigPanel extends javax.swing.JPanel { } }); + skipKnownFilesCheckbox.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchModuleSettingsPanel.class, "FileExtMismatchModuleSettingsPanel.skipKnownFilesCheckbox.text")); // NOI18N + skipKnownFilesCheckbox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + skipKnownFilesCheckboxActionPerformed(evt); + } + }); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -70,7 +90,8 @@ class FileExtMismatchSimpleConfigPanel extends javax.swing.JPanel { .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(skipTextPlain) - .addComponent(skipNoExtCheckBox)) + .addComponent(skipNoExtCheckBox) + .addComponent(skipKnownFilesCheckbox)) .addGap(0, 138, Short.MAX_VALUE)) ); layout.setVerticalGroup( @@ -79,20 +100,26 @@ class FileExtMismatchSimpleConfigPanel extends javax.swing.JPanel { .addComponent(skipNoExtCheckBox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(skipTextPlain) - .addContainerGap(51, Short.MAX_VALUE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(skipKnownFilesCheckbox) + .addContainerGap(28, Short.MAX_VALUE)) ); }// //GEN-END:initComponents private void skipNoExtCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipNoExtCheckBoxActionPerformed - FileExtMismatchIngestModule.setSkipNoExt(skipNoExtCheckBox.isSelected()); + settings.setSkipFilesWithNoExtension(skipNoExtCheckBox.isSelected()); }//GEN-LAST:event_skipNoExtCheckBoxActionPerformed private void skipTextPlainActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipTextPlainActionPerformed - FileExtMismatchIngestModule.setSkipTextPlain(skipTextPlain.isSelected()); - + settings.setSkipFilesWithTextPlainMimeType(skipTextPlain.isSelected()); }//GEN-LAST:event_skipTextPlainActionPerformed + private void skipKnownFilesCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipKnownFilesCheckboxActionPerformed + settings.setSkipKnownFiles(skipKnownFilesCheckbox.isSelected()); + }//GEN-LAST:event_skipKnownFilesCheckboxActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JCheckBox skipKnownFilesCheckbox; private javax.swing.JCheckBox skipNoExtCheckBox; private javax.swing.JCheckBox skipTextPlain; // End of variables declaration//GEN-END:variables diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java index 670d6040e5..ba4f85d0b5 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchOptionsPanelController.java @@ -16,19 +16,18 @@ import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; @OptionsPanelController.TopLevelRegistration( - categoryName = "#OptionsCategory_Name_FileExtMismatchOptions", -iconBase = "org/sleuthkit/autopsy/fileextmismatch/options-icon.png", -position = 4, -keywords = "#OptionsCategory_FileExtMismatch", -keywordsCategory = "KeywordSearchOptions") -// migrated to Bundle -//@org.openide.util.NbBundle.Messages({"OptionsCategory_Name_FileExtMismatchOptions=File Ext Mismatch", "OptionsCategory_FileExtMismatch=File Ext Mismatch"}) + categoryName = "#OptionsCategory_Name_FileExtMismatchOptions", + iconBase = "org/sleuthkit/autopsy/fileextmismatch/options-icon.png", + position = 4, + keywords = "#OptionsCategory_FileExtMismatch", + keywordsCategory = "KeywordSearchOptions") public final class FileExtMismatchOptionsPanelController extends OptionsPanelController { - private FileExtMismatchConfigPanel panel; + private FileExtMismatchSettingsPanel panel; private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); private boolean changed; private static final Logger logger = Logger.getLogger(FileExtMismatchOptionsPanelController.class.getName()); + @Override public void update() { getPanel().load(); @@ -77,9 +76,9 @@ public final class FileExtMismatchOptionsPanelController extends OptionsPanelCon pcs.removePropertyChangeListener(l); } - private FileExtMismatchConfigPanel getPanel() { + private FileExtMismatchSettingsPanel getPanel() { if (panel == null) { - panel = new FileExtMismatchConfigPanel(); + panel = new FileExtMismatchSettingsPanel(); } return panel; } @@ -87,11 +86,10 @@ public final class FileExtMismatchOptionsPanelController extends OptionsPanelCon void changed() { if (!changed) { changed = true; - + try { pcs.firePropertyChange(OptionsPanelController.PROP_CHANGED, false, true); - } - catch (Exception e) { + } catch (Exception e) { logger.log(Level.SEVERE, "FileExtMismatchOptionsPanelController listener threw exception", e); MessageNotifyUtil.Notify.show( NbBundle.getMessage(this.getClass(), "FileExtMismatchOptionsPanelController.moduleErr"), @@ -99,16 +97,15 @@ public final class FileExtMismatchOptionsPanelController extends OptionsPanelCon MessageNotifyUtil.MessageType.ERROR); } } - - try { - pcs.firePropertyChange(OptionsPanelController.PROP_VALID, null, null); - } - catch (Exception e) { - logger.log(Level.SEVERE, "FileExtMismatchOptionsPanelController listener threw exception", e); - MessageNotifyUtil.Notify.show( - NbBundle.getMessage(this.getClass(), "FileExtMismatchOptionsPanelController.moduleErr"), - NbBundle.getMessage(this.getClass(), "FileExtMismatchOptionsPanelController.moduleErr.msg"), - MessageNotifyUtil.MessageType.ERROR); - } + + try { + pcs.firePropertyChange(OptionsPanelController.PROP_VALID, null, null); + } catch (Exception e) { + logger.log(Level.SEVERE, "FileExtMismatchOptionsPanelController listener threw exception", e); + MessageNotifyUtil.Notify.show( + NbBundle.getMessage(this.getClass(), "FileExtMismatchOptionsPanelController.moduleErr"), + NbBundle.getMessage(this.getClass(), "FileExtMismatchOptionsPanelController.moduleErr.msg"), + MessageNotifyUtil.MessageType.ERROR); + } } } diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchConfigPanel.form b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.form similarity index 89% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchConfigPanel.form rename to FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.form index 9ae257607d..299bacab53 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchConfigPanel.form +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.form @@ -53,7 +53,7 @@ - + @@ -128,7 +128,7 @@ - + @@ -148,7 +148,7 @@ - + @@ -158,7 +158,7 @@ - + @@ -168,7 +168,7 @@ - + @@ -181,14 +181,14 @@ - + - + @@ -254,7 +254,7 @@ - + @@ -264,7 +264,7 @@ - + @@ -287,7 +287,7 @@ - + @@ -297,7 +297,7 @@ - + @@ -307,14 +307,14 @@ - + - + @@ -328,7 +328,7 @@ - + diff --git a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchConfigPanel.java b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.java similarity index 86% rename from FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchConfigPanel.java rename to FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.java index 77fb8f1bb6..98d701cbca 100644 --- a/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchConfigPanel.java +++ b/FileExtMismatch/src/org/sleuthkit/autopsy/fileextmismatch/FileExtMismatchSettingsPanel.java @@ -16,7 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.fileextmismatch; import java.awt.Color; @@ -30,50 +29,51 @@ import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; import javax.swing.table.AbstractTableModel; - +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.filetypeid.FileTypeIdIngestModule; +import org.sleuthkit.autopsy.corecomponents.OptionsPanel; /** - * Container panel for File Extension Mismatch Ingest Module advanced configuration options + * Container panel for File Extension Mismatch Ingest Module advanced + * configuration options */ -final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements OptionsPanel { - private static Logger logger = Logger.getLogger(FileExtMismatchConfigPanel.class.getName()); +final class FileExtMismatchSettingsPanel extends IngestModuleGlobalSetttingsPanel implements OptionsPanel { + + private static Logger logger = Logger.getLogger(FileExtMismatchSettingsPanel.class.getName()); private HashMap editableMap = new HashMap<>(); private ArrayList mimeList = null; private ArrayList currentExtensions = null; private MimeTableModel mimeTableModel; private ExtTableModel extTableModel; - private final String EXT_HEADER_LABEL = NbBundle.getMessage(FileExtMismatchConfigPanel.class, - "AddFileExtensionAction.extHeaderLbl.text"); + private final String EXT_HEADER_LABEL = NbBundle.getMessage(FileExtMismatchSettingsPanel.class, + "AddFileExtensionAction.extHeaderLbl.text"); private String selectedMime = ""; private String selectedExt = ""; ListSelectionModel lsm = null; - - public FileExtMismatchConfigPanel() { + + public FileExtMismatchSettingsPanel() { mimeTableModel = new MimeTableModel(); extTableModel = new ExtTableModel(); - + initComponents(); customizeComponents(); } - + private void customizeComponents() { setName(NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.name.text")); - + // Handle selections on the left table lsm = mimeTable.getSelectionModel(); - lsm.addListSelectionListener(new ListSelectionListener() { - + lsm.addListSelectionListener(new ListSelectionListener() { @Override public void valueChanged(ListSelectionEvent e) { ListSelectionModel listSelectionModel = (ListSelectionModel) e.getSource(); if (!listSelectionModel.isSelectionEmpty()) { int index = listSelectionModel.getMinSelectionIndex(); listSelectionModel.setSelectionInterval(index, index); - + selectedMime = mimeList.get(index); String labelStr = EXT_HEADER_LABEL + selectedMime + ":"; if (labelStr.length() > 80) { @@ -81,7 +81,7 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt } extHeaderLabel.setText(labelStr); updateExtList(); - + extTableModel.resync(); //initButtons(); } else { @@ -89,32 +89,31 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt currentExtensions = null; extTableModel.resync(); } - + clearErrLabels(); - } + } }); - + // Handle selections on the right table ListSelectionModel extLsm = extTable.getSelectionModel(); - extLsm.addListSelectionListener(new ListSelectionListener() { - + extLsm.addListSelectionListener(new ListSelectionListener() { @Override public void valueChanged(ListSelectionEvent e) { ListSelectionModel listSelectionModel = (ListSelectionModel) e.getSource(); if (!listSelectionModel.isSelectionEmpty()) { int index = listSelectionModel.getMinSelectionIndex(); listSelectionModel.setSelectionInterval(index, index); - + selectedExt = currentExtensions.get(index); } else { selectedExt = ""; } - + extRemoveErrLabel.setText(" "); - - } - }); - + + } + }); + } private void clearErrLabels() { @@ -122,9 +121,9 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt mimeRemoveErrLabel.setText(" "); extRemoveErrLabel.setText(" "); extErrorLabel.setText(" "); - saveMsgLabel.setText(" "); - } - + saveMsgLabel.setText(" "); + } + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -156,7 +155,7 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt saveMsgLabel = new javax.swing.JLabel(); saveButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/fileextmismatch/save16.png"))); // NOI18N - saveButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.saveButton.text")); // NOI18N + saveButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.saveButton.text")); // NOI18N saveButton.setEnabled(false); saveButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -166,26 +165,26 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt jSplitPane1.setDividerLocation(430); - jLabel1.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.jLabel1.text")); // NOI18N + jLabel1.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.jLabel1.text")); // NOI18N mimeTable.setModel(mimeTableModel); jScrollPane2.setViewportView(mimeTable); - userTypeTextField.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.userTypeTextField.text")); // NOI18N + userTypeTextField.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.userTypeTextField.text")); // NOI18N userTypeTextField.addFocusListener(new java.awt.event.FocusAdapter() { public void focusGained(java.awt.event.FocusEvent evt) { userTypeTextFieldFocusGained(evt); } }); - addTypeButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.addTypeButton.text")); // NOI18N + addTypeButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.addTypeButton.text")); // NOI18N addTypeButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { addTypeButtonActionPerformed(evt); } }); - removeTypeButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.removeTypeButton.text")); // NOI18N + removeTypeButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.removeTypeButton.text")); // NOI18N removeTypeButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { removeTypeButtonActionPerformed(evt); @@ -193,9 +192,9 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt }); mimeErrLabel.setForeground(new java.awt.Color(255, 0, 0)); - mimeErrLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.mimeErrLabel.text")); // NOI18N + mimeErrLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.mimeErrLabel.text")); // NOI18N - mimeRemoveErrLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.mimeRemoveErrLabel.text")); // NOI18N + mimeRemoveErrLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.mimeRemoveErrLabel.text")); // NOI18N javax.swing.GroupLayout mimePanelLayout = new javax.swing.GroupLayout(mimePanel); mimePanel.setLayout(mimePanelLayout); @@ -240,14 +239,14 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt jSplitPane1.setLeftComponent(mimePanel); - userExtTextField.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.userExtTextField.text")); // NOI18N + userExtTextField.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.userExtTextField.text")); // NOI18N userExtTextField.addFocusListener(new java.awt.event.FocusAdapter() { public void focusGained(java.awt.event.FocusEvent evt) { userExtTextFieldFocusGained(evt); } }); - addExtButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.addExtButton.text")); // NOI18N + addExtButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.addExtButton.text")); // NOI18N addExtButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { addExtButtonActionPerformed(evt); @@ -257,19 +256,19 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt extTable.setModel(extTableModel); jScrollPane3.setViewportView(extTable); - removeExtButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.removeExtButton.text")); // NOI18N + removeExtButton.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.removeExtButton.text")); // NOI18N removeExtButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { removeExtButtonActionPerformed(evt); } }); - extHeaderLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.extHeaderLabel.text")); // NOI18N + extHeaderLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.extHeaderLabel.text")); // NOI18N extErrorLabel.setForeground(new java.awt.Color(255, 0, 0)); - extErrorLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.extErrorLabel.text")); // NOI18N + extErrorLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.extErrorLabel.text")); // NOI18N - extRemoveErrLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.extRemoveErrLabel.text")); // NOI18N + extRemoveErrLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.extRemoveErrLabel.text")); // NOI18N javax.swing.GroupLayout extensionPanelLayout = new javax.swing.GroupLayout(extensionPanel); extensionPanel.setLayout(extensionPanelLayout); @@ -315,7 +314,7 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt jSplitPane1.setRightComponent(extensionPanel); saveMsgLabel.setForeground(new java.awt.Color(0, 0, 255)); - saveMsgLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchConfigPanel.class, "FileExtMismatchConfigPanel.saveMsgLabel.text")); // NOI18N + saveMsgLabel.setText(org.openide.util.NbBundle.getMessage(FileExtMismatchSettingsPanel.class, "FileExtMismatchSettingsPanel.saveMsgLabel.text")); // NOI18N javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -349,41 +348,41 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt private void addExtButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_addExtButtonActionPerformed String newExt = userExtTextField.getText(); if (newExt.isEmpty()) { - extErrorLabel.setForeground(Color.red); + extErrorLabel.setForeground(Color.red); extErrorLabel.setText( NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.addExtButton.errLabel.empty")); return; } - + if (selectedMime.isEmpty()) { - extErrorLabel.setForeground(Color.red); + extErrorLabel.setForeground(Color.red); extErrorLabel.setText( NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.addExtButton.errLabel.noMimeType")); return; } - + if (currentExtensions.contains(newExt)) { extErrorLabel.setForeground(Color.red); extErrorLabel.setText( NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.addExtButton.errLabel.extExists")); - return; - } - - ArrayList editedExtensions = new ArrayList<>(Arrays.asList(editableMap.get(selectedMime))); + return; + } + + ArrayList editedExtensions = new ArrayList<>(Arrays.asList(editableMap.get(selectedMime))); editedExtensions.add(newExt); - + // Old array will be replaced by new array for this key - editableMap.put(selectedMime, editedExtensions.toArray(new String[0])); + editableMap.put(selectedMime, editedExtensions.toArray(new String[0])); // Refresh table - updateExtList(); + updateExtList(); extTableModel.resync(); - + // user feedback for successful add extErrorLabel.setForeground(Color.blue); extErrorLabel.setText( NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.addExtButton.errLabel.extAdded", - newExt)); + newExt)); extRemoveErrLabel.setText(" "); userExtTextField.setText(""); setIsModified(); @@ -400,26 +399,26 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt mimeErrLabel.setText(NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.addTypeButton.empty")); return; } - if (newMime.equals( "application/octet-stream")){ + if (newMime.equals("application/octet-stream")) { mimeErrLabel.setForeground(Color.red); mimeErrLabel.setText(NbBundle.getMessage(this.getClass(), - "FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotSupported")); - return; + "FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotSupported")); + return; } if (mimeList.contains(newMime)) { mimeErrLabel.setForeground(Color.red); mimeErrLabel.setText( NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.addTypeButton.mimeTypeExists")); - return; + return; } - + if (!FileTypeIdIngestModule.isMimeTypeDetectable(newMime)) { mimeErrLabel.setForeground(Color.red); mimeErrLabel.setText(NbBundle.getMessage(this.getClass(), - "FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotDetectable")); - return; + "FileExtMismatchConfigPanel.addTypeButton.mimeTypeNotDetectable")); + return; } - + editableMap.put(newMime, new String[0]); // Refresh table @@ -450,15 +449,15 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt mimeRemoveErrLabel.setText( NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.removeTypeButton.noneSelected")); return; - } - - editableMap.remove(selectedMime); + } + + editableMap.remove(selectedMime); String deadMime = selectedMime; - + // Refresh table updateMimeList(); - mimeTableModel.resync(); - + mimeTableModel.resync(); + // user feedback for successful add mimeRemoveErrLabel.setForeground(Color.blue); mimeRemoveErrLabel.setText( @@ -472,26 +471,26 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt extRemoveErrLabel.setText( NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.removeExtButton.noneSelected")); return; - } - + } + if (selectedMime.isEmpty()) { - extErrorLabel.setForeground(Color.red); + extErrorLabel.setForeground(Color.red); extErrorLabel.setText(NbBundle.getMessage(this.getClass(), - "FileExtMismatchConfigPanel.removeExtButton.noMimeTypeSelected")); + "FileExtMismatchConfigPanel.removeExtButton.noMimeTypeSelected")); return; - } - - ArrayList editedExtensions = new ArrayList<>(Arrays.asList(editableMap.get(selectedMime))); + } + + ArrayList editedExtensions = new ArrayList<>(Arrays.asList(editableMap.get(selectedMime))); editedExtensions.remove(selectedExt); String deadExt = selectedExt; - + // Old array will be replaced by new array for this key - editableMap.put(selectedMime, editedExtensions.toArray(new String[0])); - + editableMap.put(selectedMime, editedExtensions.toArray(new String[0])); + // Refresh tables updateExtList(); - extTableModel.resync(); - + extTableModel.resync(); + // user feedback for successful add extRemoveErrLabel.setForeground(Color.blue); extRemoveErrLabel.setText( @@ -505,7 +504,7 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt Collections.sort(mimeList); } } - + private void updateExtList() { String[] temp = editableMap.get(selectedMime); if (temp != null) { @@ -517,68 +516,72 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt currentExtensions = null; } } - + + @Override + public void saveSettings() { + if (FileExtMismatchXML.getDefault().save(editableMap)) { + mimeErrLabel.setText(" "); + mimeRemoveErrLabel.setText(" "); + extRemoveErrLabel.setText(" "); + extErrorLabel.setText(" "); + + saveMsgLabel.setText(NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.store.msg")); + saveButton.setEnabled(false); + } else { + //error + JOptionPane.showMessageDialog(this, + NbBundle.getMessage(this.getClass(), + "FileExtMismatchConfigPanel.store.msgDlg.msg"), + NbBundle.getMessage(this.getClass(), + "FileExtMismatchConfigPanel.save.msgDlg.title"), + JOptionPane.ERROR_MESSAGE); + } + } + @Override public void load() { // Load the XML into a buffer that the user can modify. They can choose // to save it back to the file after making changes. editableMap = FileExtMismatchXML.getDefault().load(); updateMimeList(); - updateExtList(); + updateExtList(); } @Override public void store() { - if (FileExtMismatchXML.getDefault().save(editableMap)) { - mimeErrLabel.setText(" "); - mimeRemoveErrLabel.setText(" "); - extRemoveErrLabel.setText(" "); - extErrorLabel.setText(" "); - - saveMsgLabel.setText(NbBundle.getMessage(this.getClass(), "FileExtMismatchConfigPanel.store.msg")); - saveButton.setEnabled(false); - } else { - //error - JOptionPane.showMessageDialog(this, - NbBundle.getMessage(this.getClass(), - "FileExtMismatchConfigPanel.store.msgDlg.msg"), - NbBundle.getMessage(this.getClass(), - "FileExtMismatchConfigPanel.save.msgDlg.title"), - JOptionPane.ERROR_MESSAGE); - } + saveSettings(); } private void setIsModified() { saveButton.setEnabled(true); saveMsgLabel.setText(" "); } - + public void cancel() { clearErrLabels(); load(); // The next time this panel is opened, we want it to be fresh } - + public void ok() { // if data is unsaved if (saveButton.isEnabled()) { - int choice = JOptionPane.showConfirmDialog(this, - NbBundle.getMessage(this.getClass(), - "FileExtMismatchConfigPanel.ok.confDlg.msg"), - NbBundle.getMessage(this.getClass(), - "FileExtMismatchConfigPanel.confDlg.title"), - JOptionPane.YES_NO_OPTION); - if (choice == JOptionPane.YES_OPTION) { - store(); - } + int choice = JOptionPane.showConfirmDialog(this, + NbBundle.getMessage(this.getClass(), + "FileExtMismatchConfigPanel.ok.confDlg.msg"), + NbBundle.getMessage(this.getClass(), + "FileExtMismatchConfigPanel.confDlg.title"), + JOptionPane.YES_NO_OPTION); + if (choice == JOptionPane.YES_OPTION) { + store(); + } } - clearErrLabels(); + clearErrLabels(); load(); // The next time this panel is opened, we want it to be fresh } - + boolean valid() { return true; } - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton addExtButton; private javax.swing.JButton addTypeButton; @@ -666,7 +669,7 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt fireTableDataChanged(); } } - + private class ExtTableModel extends AbstractTableModel { @Override @@ -697,7 +700,7 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt @Override public Object getValueAt(int rowIndex, int columnIndex) { Object ret = null; - + if ((currentExtensions == null) || (currentExtensions.size() == 0) || (rowIndex > currentExtensions.size())) { return ""; } @@ -730,6 +733,5 @@ final class FileExtMismatchConfigPanel extends javax.swing.JPanel implements Opt void resync() { fireTableDataChanged(); } - } - + } } diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle.properties b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle.properties index 9aa6bb3854..4850085f1e 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle.properties +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle.properties @@ -1,8 +1,8 @@ OpenIDE-Module-Name=FileTypeId -FileTypeIdSimpleConfigPanel.skipKnownCheckBox.toolTipText=Depending on how many files have known hashes, checking this box will improve the speed of file type identification. -FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text=Skip Known Files (NSRL) +FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text=Skip known files (NSRL) +FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText=Depending on how many files have known hashes, checking this box will improve the speed of file type identification. FileTypeIdIngestModule.moduleName.text=File Type Identification FileTypeIdIngestModule.moduleDesc.text=Matches file types based on binary signatures. FileTypeIdIngestModule.complete.totalProcTime=Total Processing Time FileTypeIdIngestModule.complete.totalFiles=Total Files Processed -FileTypeIdIngestModule.complete.srvMsg.text=File Type Id Results \ No newline at end of file +FileTypeIdIngestModule.complete.srvMsg.text=File Type Id Results diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle_ja.properties b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle_ja.properties index 518546267c..047da33077 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle_ja.properties +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/Bundle_ja.properties @@ -1,8 +1,7 @@ -OpenIDE-Module-Name=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A -FileTypeIdSimpleConfigPanel.skipKnownCheckBox.toolTipText=\u65E2\u77E5\u306E\u30CF\u30C3\u30B7\u30E5\u5024\u3092\u6301\u3064\u30D5\u30A1\u30A4\u30EB\u6570\u306B\u3088\u3063\u3066\u306F\u3001\u3053\u306E\u30DC\u30C3\u30AF\u30B9\u3092\u9078\u629E\u3059\u308B\u306E\u306B\u3088\u308A\u3001\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A\u3092\u52A0\u901F\u3057\u307E\u3059\u3002 -FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text=\u65E2\u77E5\u30D5\u30A1\u30A4\u30EB\u3092\u30B9\u30AD\u30C3\u30D7(NSRL) -FileTypeIdIngestModule.moduleName.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u306E\u7279\u5B9A -FileTypeIdIngestModule.moduleDesc.text=\u30D0\u30A4\u30CA\u30EA\u7F72\u540D\u306B\u57FA\u3065\u3044\u3066\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u3092\u4E00\u81F4\u3059\u308B\u3002 -FileTypeIdIngestModule.complete.totalProcTime=\u5408\u8A08\u51E6\u7406\u6642\u9593 -FileTypeIdIngestModule.complete.totalFiles=\u5408\u8A08\u51E6\u7406\u30D5\u30A1\u30A4\u30EB\u6570 -FileTypeIdIngestModule.complete.srvMsg.text=\u30D5\u30A1\u30A4\u30EB\u30BF\u30A4\u30D7\u7279\u5B9A\u306E\u7D50\u679C \ No newline at end of file +OpenIDE-Module-Name=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a +FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText=\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u6301\u3064\u30d5\u30a1\u30a4\u30eb\u6570\u306b\u3088\u3063\u3066\u306f\u3001\u3053\u306e\u30dc\u30c3\u30af\u30b9\u3092\u9078\u629e\u3059\u308b\u306e\u306b\u3088\u308a\u3001\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a\u3092\u52a0\u901f\u3057\u307e\u3059\u3002 +FileTypeIdIngestModule.moduleName.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u306e\u7279\u5b9a +FileTypeIdIngestModule.moduleDesc.text=\u30d0\u30a4\u30ca\u30ea\u7f72\u540d\u306b\u57fa\u3065\u3044\u3066\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u3092\u4e00\u81f4\u3059\u308b\u3002 +FileTypeIdIngestModule.complete.totalProcTime=\u5408\u8a08\u51e6\u7406\u6642\u9593 +FileTypeIdIngestModule.complete.totalFiles=\u5408\u8a08\u51e6\u7406\u30d5\u30a1\u30a4\u30eb\u6570 +FileTypeIdIngestModule.complete.srvMsg.text=\u30d5\u30a1\u30a4\u30eb\u30bf\u30a4\u30d7\u7279\u5b9a\u306e\u7d50\u679c \ No newline at end of file diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java index a28a10ded7..50331c3a88 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2013 Basis Technology Corp. + * Copyright 2013-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,16 +27,15 @@ import org.sleuthkit.datamodel.AbstractFile; * to an extension string list instead of the third-party library's extension * reporting. */ - interface FileTypeDetectionInterface { - - // Struct to hold multiple values for return +interface FileTypeDetectionInterface { + public class FileIdInfo { - public String type; - public String extension; + + public String type; + public String extension; } - - // You only have one job + FileIdInfo attemptMatch(AbstractFile abstractFile); - + boolean isMimeTypeDetectable(String mimeType); } diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java index a0f3afd001..858a4a7d7f 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2013 Basis Technology Corp. + * Copyright 2013-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,19 +16,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.filetypeid; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestMessage; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; -import org.sleuthkit.autopsy.ingest.IngestModuleInit; import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.autopsy.ingest.PipelineContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -36,84 +32,57 @@ import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; +import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; +import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; /** - * Detects the type of a file based on signature (magic) values. - * Posts results to the blackboard. + * Detects the type of a file based on signature (magic) values. Posts results + * to the blackboard. */ - public class FileTypeIdIngestModule extends org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile { - private static FileTypeIdIngestModule defaultInstance = null; - public final static String MODULE_NAME = NbBundle.getMessage(FileTypeIdIngestModule.class, - "FileTypeIdIngestModule.moduleName.text"); - public final static String MODULE_DESCRIPTION = NbBundle.getMessage(FileTypeIdIngestModule.class, - "FileTypeIdIngestModule.moduleDesc.text"); - public final static String MODULE_VERSION = Version.getVersion(); +public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileIngestModule { + private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); - private static long matchTime = 0; - private static int messageId = 0; - private static long numFiles = 0; - // NOTE: This value needs to be in sync with the default GUI value - // given the current design of only updating this when the user changes the default. - private static boolean skipKnown = true; - private static long MIN_FILE_SIZE = 512; - - private FileTypeIdSimpleConfigPanel simpleConfigPanel; - private IngestServices services; - + private static final long MIN_FILE_SIZE = 512; + private final FileTypeIdModuleSettings settings; + private long matchTime = 0; + private int messageId = 0; // RJCTODO: If this is not made a thread safe static, duplicate message ids will be used + private long numFiles = 0; // The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed. // If desired in the future to be more knowledgable about weird files or rare formats, we could // actually have a list of detectors which are called in order until a match is found. - private FileTypeDetectionInterface detector = new TikaFileTypeDetector(); - //private FileTypeDetectionInterface detector = new JMimeMagicFileTypeDetector(); - //private FileTypeDetectionInterface detector = new MimeUtilFileTypeDetector(); - - // Private to ensure Singleton status - private FileTypeIdIngestModule() { - } - - // File-level ingest modules are currently singleton -- this is required - public static synchronized FileTypeIdIngestModule getDefault() { - //defaultInstance is a private static class variable - if (defaultInstance == null) { - defaultInstance = new FileTypeIdIngestModule(); - } - return defaultInstance; + private FileTypeDetectionInterface detector = new TikaFileTypeDetector(); + + FileTypeIdIngestModule(FileTypeIdModuleSettings settings) { + this.settings = settings; } - @Override - public void init(IngestModuleInit initContext) throws IngestModuleException { - services = IngestServices.getDefault(); - } - - @Override - public ProcessResult process(PipelineContext pipelineContext, AbstractFile abstractFile) { + public ProcessResult process(AbstractFile abstractFile) { // skip non-files - if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) || - (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { - + if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) { + return ProcessResult.OK; } - - if (skipKnown && (abstractFile.getKnown() == FileKnown.KNOWN)) { + + if (settings.skipKnownFiles() && (abstractFile.getKnown() == FileKnown.KNOWN)) { return ProcessResult.OK; } - + if (abstractFile.getSize() < MIN_FILE_SIZE) { - return ProcessResult.OK; - } - - try - { + return ProcessResult.OK; + } + + try { long startTime = System.currentTimeMillis(); FileTypeDetectionInterface.FileIdInfo fileId = detector.attemptMatch(abstractFile); matchTime += (System.currentTimeMillis() - startTime); numFiles++; - + if (!fileId.type.isEmpty()) { // add artifact BlackboardArtifact bart = abstractFile.getGenInfoArtifact(); - BlackboardAttribute batt = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID(), MODULE_NAME, fileId.type); + BlackboardAttribute batt = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID(), FileTypeIdModuleFactory.getModuleName(), fileId.type); bart.addAttribute(batt); // we don't fire the event because we just updated TSK_GEN_INFO, which isn't displayed in the tree and is vague. @@ -122,88 +91,38 @@ import org.sleuthkit.datamodel.TskException; } catch (TskException ex) { logger.log(Level.WARNING, "Error matching file signature", ex); return ProcessResult.ERROR; - } - catch (Exception e) { + } catch (Exception e) { logger.log(Level.WARNING, "Error matching file signature", e); return ProcessResult.ERROR; } } - @Override - public void complete() { + public void shutDown(boolean ingestJobCancelled) { StringBuilder detailsSb = new StringBuilder(); - //details detailsSb.append(""); - - detailsSb.append(""); - + detailsSb.append(""); detailsSb.append("\n"); + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) + .append("\n"); detailsSb.append("\n"); + .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) + .append("\n"); detailsSb.append("
"+MODULE_DESCRIPTION+"
").append(FileTypeIdModuleFactory.getModuleName()).append("
") - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) - .append("").append(matchTime).append("
").append(matchTime).append("
") - .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) - .append("").append(numFiles).append("
").append(numFiles).append("
"); - - services.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, this, - NbBundle.getMessage(this.getClass(), - "FileTypeIdIngestModule.complete.srvMsg.text"), - detailsSb.toString())); + IngestServices.getDefault().postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "FileTypeIdIngestModule.complete.srvMsg.text"), + detailsSb.toString())); } - @Override - public void stop() { - //do nothing - } - - @Override - public String getName() { - return MODULE_NAME; - } - - @Override - public String getDescription() { - return MODULE_DESCRIPTION; - } - - @Override - public String getVersion() { - return MODULE_VERSION; - } - - @Override - public boolean hasSimpleConfiguration() { - return true; - } - - @Override - public javax.swing.JPanel getSimpleConfiguration(String context) { - if (simpleConfigPanel == null) { - simpleConfigPanel = new FileTypeIdSimpleConfigPanel(); - } - - return simpleConfigPanel; - } - - @Override - public boolean hasBackgroundJobsRunning() { - // we're single threaded... - return false; - } - - protected static void setSkipKnown(boolean flag) { - skipKnown = flag; - } - /** * Validate if a given mime type is in the detector's registry. + * * @param mimeType Full string of mime type, e.g. "text/html" * @return true if detectable */ public static boolean isMimeTypeDetectable(String mimeType) { - FileTypeDetectionInterface detector = new TikaFileTypeDetector(); + FileTypeDetectionInterface detector = new TikaFileTypeDetector(); return detector.isMimeTypeDetectable(mimeType); - } - + } } \ No newline at end of file diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleFactory.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleFactory.java new file mode 100755 index 0000000000..5b8e27a2b9 --- /dev/null +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleFactory.java @@ -0,0 +1,90 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.filetypeid; + +import org.openide.util.NbBundle; +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleFactory; +import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleSettingsPanel; + +/** + * An factory that creates file ingest modules that determine the types of + * files. + */ +@ServiceProvider(service = IngestModuleFactory.class) +public class FileTypeIdModuleFactory extends IngestModuleFactoryAdapter { + + @Override + public String getModuleDisplayName() { + return getModuleName(); + } + + static String getModuleName() { + return NbBundle.getMessage(FileTypeIdIngestModule.class, + "FileTypeIdIngestModule.moduleName.text"); + } + + @Override + public String getModuleDescription() { + return NbBundle.getMessage(FileTypeIdIngestModule.class, + "FileTypeIdIngestModule.moduleDesc.text"); + } + + @Override + public String getModuleVersionNumber() { + return Version.getVersion(); + } + + @Override + public IngestModuleSettings getDefaultModuleSettings() { + return new FileTypeIdModuleSettings(); + } + + @Override + public boolean hasModuleSettingsPanel() { + return true; + } + + @Override + public IngestModuleSettingsPanel getModuleSettingsPanel(IngestModuleSettings settings) { + assert settings instanceof FileTypeIdModuleSettings; + if (!(settings instanceof FileTypeIdModuleSettings)) { + throw new IllegalArgumentException("Expected settings argument to be instanceof FileTypeIdModuleSettings"); + } + return new FileTypeIdModuleSettingsPanel((FileTypeIdModuleSettings) settings); + } + + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleSettings settings) { + assert settings instanceof FileTypeIdModuleSettings; + if (!(settings instanceof FileTypeIdModuleSettings)) { + throw new IllegalArgumentException("Expected settings argument to be instanceof FileTypeIdModuleSettings"); + } + return new FileTypeIdIngestModule((FileTypeIdModuleSettings) settings); + } +} diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettings.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettings.java new file mode 100755 index 0000000000..d5a962c8d4 --- /dev/null +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettings.java @@ -0,0 +1,44 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.filetypeid; + +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; + +/** + * Ingest job options for the file type identifier ingest module instances. + */ +public class FileTypeIdModuleSettings implements IngestModuleSettings { + + private boolean skipKnownFiles = true; + + FileTypeIdModuleSettings() { + } + + FileTypeIdModuleSettings(boolean skipKnownFiles) { + this.skipKnownFiles = skipKnownFiles; + } + + void setSkipKnownFiles(boolean enabled) { + skipKnownFiles = enabled; + } + + boolean skipKnownFiles() { + return skipKnownFiles; + } +} \ No newline at end of file diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.form similarity index 74% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form rename to FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.form index b91cd2e25e..da4b241fb4 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.form @@ -19,7 +19,7 @@ - + @@ -28,7 +28,7 @@ - + @@ -38,10 +38,13 @@ - + - + + + + diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.java similarity index 64% rename from FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java rename to FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.java index 980f4e4f41..cd4092f07d 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdModuleSettingsPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 - 2013 Basis Technology Corp. + * Copyright 2013 - 2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,25 +18,36 @@ */ package org.sleuthkit.autopsy.filetypeid; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleSettingsPanel; /** - * Instances of this class provide a simplified UI for managing the hash sets configuration. + * UI component used to set ingest job options for file type identifier ingest + * modules. */ - class FileTypeIdSimpleConfigPanel extends javax.swing.JPanel { +final class FileTypeIdModuleSettingsPanel extends IngestModuleSettingsPanel { - public FileTypeIdSimpleConfigPanel() { + private final FileTypeIdModuleSettings settings; + + public FileTypeIdModuleSettingsPanel(FileTypeIdModuleSettings settings) { + this.settings = settings; initComponents(); customizeComponents(); } - + private void customizeComponents() { + skipKnownCheckBox.setSelected(settings.skipKnownFiles()); } - - - /** This method is called from within the constructor to - * initialize the form. - * WARNING: Do NOT modify this code. The content of this method is - * always regenerated by the Form Editor. + + @Override + public IngestModuleSettings getSettings() { + return settings; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. */ @SuppressWarnings("unchecked") // //GEN-BEGIN:initComponents @@ -45,8 +56,9 @@ package org.sleuthkit.autopsy.filetypeid; skipKnownCheckBox = new javax.swing.JCheckBox(); skipKnownCheckBox.setSelected(true); - skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text")); // NOI18N - skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.toolTipText")); // NOI18N + skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdModuleSettingsPanel.class, "FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text")); // NOI18N + skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdModuleSettingsPanel.class, "FileTypeIdModuleSettingsPanel.skipKnownCheckBox.toolTipText")); // NOI18N + skipKnownCheckBox.setLabel(org.openide.util.NbBundle.getMessage(FileTypeIdModuleSettingsPanel.class, "FileTypeIdModuleSettingsPanel.skipKnownCheckBox.text")); // NOI18N skipKnownCheckBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { skipKnownCheckBoxActionPerformed(evt); @@ -60,21 +72,20 @@ package org.sleuthkit.autopsy.filetypeid; .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) .addComponent(skipKnownCheckBox) - .addContainerGap(138, Short.MAX_VALUE)) + .addContainerGap(608, Short.MAX_VALUE)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(11, 11, 11) .addComponent(skipKnownCheckBox) - .addContainerGap(45, Short.MAX_VALUE)) + .addContainerGap(47, Short.MAX_VALUE)) ); }// //GEN-END:initComponents private void skipKnownCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipKnownCheckBoxActionPerformed - FileTypeIdIngestModule.setSkipKnown(skipKnownCheckBox.isSelected()); + settings.setSkipKnownFiles(skipKnownCheckBox.isSelected()); }//GEN-LAST:event_skipKnownCheckBoxActionPerformed - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JCheckBox skipKnownCheckBox; // End of variables declaration//GEN-END:variables diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java index 8ab655e27b..e074e0f003 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 - 2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,10 +16,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.filetypeid; + import java.util.SortedSet; -import org.openide.util.Exceptions; import org.apache.tika.Tika; import org.apache.tika.mime.MediaType; import org.apache.tika.mime.MimeTypes; @@ -29,14 +28,13 @@ import org.sleuthkit.datamodel.AbstractFile; class TikaFileTypeDetector implements FileTypeDetectionInterface { private static Tika tikaInst = new Tika(); - + @Override public FileTypeDetectionInterface.FileIdInfo attemptMatch(AbstractFile abstractFile) { - try { + try { FileTypeDetectionInterface.FileIdInfo ret = new FileTypeDetectionInterface.FileIdInfo(); final int maxBytesInitial = 100; //how many bytes to read on first pass byte buffer[] = new byte[maxBytesInitial]; - int len = abstractFile.read(buffer, 0, maxBytesInitial); boolean found = false; try { @@ -66,23 +64,24 @@ class TikaFileTypeDetector implements FileTypeDetectionInterface { } catch (Exception ex) { return new FileTypeDetectionInterface.FileIdInfo(); - } + } } /** - * Validate if a given mime type is in the registry. - * For Tika, we remove the string "tika" from all MIME names, - * e.g. use "application/x-msoffice" NOT "application/x-tika-msoffice" + * Validate if a given mime type is in the registry. For Tika, we remove the + * string "tika" from all MIME names, e.g. use "application/x-msoffice" NOT + * "application/x-tika-msoffice" + * * @param mimeType Full string of mime type, e.g. "text/html" * @return true if detectable */ @Override public boolean isMimeTypeDetectable(String mimeType) { boolean ret = false; - - SortedSet m = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); + + SortedSet m = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); String[] split = mimeType.split("/"); - + if (split.length == 2) { String type = split[0]; String subtype = split[1]; @@ -90,6 +89,6 @@ class TikaFileTypeDetector implements FileTypeDetectionInterface { ret = m.contains(mediaType); } - return ret; + return ret; } } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java index 93789e0308..c94b2c95e0 100755 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java @@ -32,11 +32,12 @@ import org.openide.util.NbBundle; import org.openide.util.Utilities; import org.openide.util.Lookup; import org.openide.util.actions.Presenter; -import org.sleuthkit.autopsy.ingest.IngestConfigurator; +import org.sleuthkit.autopsy.ingest.IngestJobLauncher; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; import static org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; +import org.sleuthkit.autopsy.ingest.IngestManager; /** * Instances of this Action allow users to content to a hash database. @@ -81,8 +82,7 @@ final class AddContentToHashDbAction extends AbstractAction implements Presenter super(SINGLE_SELECTION_NAME); // Disable the menu if file ingest is in progress. - IngestConfigurator ingestConfigurator = Lookup.getDefault().lookup(IngestConfigurator.class); - if (null != ingestConfigurator && ingestConfigurator.isIngestRunning()) { + if (IngestManager.getDefault().isIngestRunning()) { setEnabled(false); return; } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle.properties b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle.properties index d5f4c47246..c4fcd6dc0d 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle.properties +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle.properties @@ -30,33 +30,10 @@ HashDbImportDatabaseDialog.cancelButton.text=Cancel HashDbCreateDatabaseDialog.jLabel2.text=Type: HashDbCreateDatabaseDialog.knownBadRadioButton.text=Known Bad HashDbCreateDatabaseDialog.cancelButton.text=Cancel -HashDbConfigPanel.nameLabel.text=Hash Set Name: -HashDbConfigPanel.hashDbNameLabel.text=No database selected -HashDbConfigPanel.hashDatabasesLabel.text=Hash Databases: -HashDbConfigPanel.hashDbLocationLabel.text=No database selected -HashDbConfigPanel.ingestWarningLabel.text=Ingest is ongoing, some settings will be unavailable until it finishes. -HashDbConfigPanel.jButton3.text=Import Database -HashDbConfigPanel.jLabel6.text=Type: -HashDbConfigPanel.jLabel4.text=Location: -HashDbConfigPanel.jLabel2.text=Name: -HashDbConfigPanel.optionsLabel.text=Options -HashDbConfigPanel.typeLabel.text=Type: -HashDbConfigPanel.locationLabel.text=Database Path: -HashDbConfigPanel.hashDbIndexStatusLabel.text=No database selected -HashDbConfigPanel.hashDbTypeLabel.text=No database selected -HashDbConfigPanel.indexButton.text=Index -HashDbConfigPanel.indexLabel.text=Index Status: -HashDbConfigPanel.informationLabel.text=Information -HashDbConfigPanel.importDatabaseButton.text=Import Database -HashDbConfigPanel.deleteDatabaseButton.text=Delete Database -HashDbConfigPanel.indexPathLabelLabel.text=Index Path: -HashDbConfigPanel.indexPathLabel.text=No database selected ModalNoButtons.CURRENTDB_LABEL.text=(CurrentDb) ModalNoButtons.CURRENTLYON_LABEL.text=Currently Indexing x of y ModalNoButtons.GO_GET_COFFEE_LABEL.text=Hash databases are currently being indexed, this may take some time. ModalNoButtons.CANCEL_BUTTON.text=Cancel -HashDbSimpleConfigPanel.knownBadHashDbsLabel.text=Select known BAD hash databases to use: -HashDbSimpleConfigPanel.knownHashDbsLabel.text=Select known hash databases to use: HashDbImportDatabaseDialog.knownRadioButton.text=Known (NSRL or other) HashDbCreateDatabaseDialog.knownRadioButton.text=Known HashDbCreateDatabaseDialog.jLabel1.text=Database Path: @@ -66,13 +43,10 @@ HashDbImportDatabaseDialog.jLabel3.text=Database Path: HashDbCreateDatabaseDialog.sendIngestMessagesCheckbox.text=Send ingest inbox messages for each hit HashDbImportDatabaseDialog.sendIngestMessagesCheckbox.text=Send ingest inbox message for each hit HashDbImportDatabaseDialog.hashSetNameTextField.text= -HashDbConfigPanel.createDatabaseButton.text=Create Database HashDbImportDatabaseDialog.openButton.text=Open... -HashDbSimpleConfigPanel.alwaysCalcHashesCheckbox.text=Calculate MD5 even if no hash database is selected HashDbCreateDatabaseDialog.jLabel3.text=Hash Set Name: HashDbCreateDatabaseDialog.okButton.text=OK HashDbCreateDatabaseDialog.databasePathTextField.text= -HashDbConfigPanel.sendIngestMessagesCheckBox.text=Send ingest inbox message for each hit AddContentToHashDbAction.ContentMenu.noHashDbsConfigd=No hash databases configured AddContentToHashDbAction.ContentMenu.createDbItem=Create database... AddContentToHashDbAction.addFilesToHashSet.addToHashDbErr=Add to Hash Database Error @@ -202,3 +176,29 @@ AddContentToHashDbAction.singleSelectionName=Add file to hash database AddContentToHashDbAction.multipleSelectionName=Add files to hash database HashDbManager.ingestRunningExceptionMsg=Ingest is ongoing; this service will be unavailable until it finishes. HashDbManager.saveErrorExceptionMsg=Error saving hash configuration +HashLookupSettingsPanel.optionsLabel.text=Options +HashLookupSettingsPanel.jButton3.text=Import Database +HashLookupSettingsPanel.indexPathLabelLabel.text=Index Path: +HashLookupSettingsPanel.createDatabaseButton.text=Create Database +HashLookupSettingsPanel.jLabel6.text=Type: +HashLookupSettingsPanel.jLabel4.text=Location: +HashLookupSettingsPanel.jLabel2.text=Name: +HashLookupSettingsPanel.indexPathLabel.text=No database selected +HashLookupSettingsPanel.ingestWarningLabel.text=Ingest is ongoing, some settings will be unavailable until it finishes. +HashLookupSettingsPanel.deleteDatabaseButton.text=Delete Database +HashLookupSettingsPanel.importDatabaseButton.text=Import Database +HashLookupSettingsPanel.hashDatabasesLabel.text=Hash Databases: +HashLookupSettingsPanel.nameLabel.text=Hash Set Name: +HashLookupSettingsPanel.informationLabel.text=Information +HashLookupSettingsPanel.sendIngestMessagesCheckBox.text=Send ingest inbox message for each hit +HashLookupSettingsPanel.hashDbLocationLabel.text=No database selected +HashLookupSettingsPanel.hashDbNameLabel.text=No database selected +HashLookupSettingsPanel.typeLabel.text=Type: +HashLookupSettingsPanel.locationLabel.text=Database Path: +HashLookupSettingsPanel.hashDbIndexStatusLabel.text=No database selected +HashLookupSettingsPanel.hashDbTypeLabel.text=No database selected +HashLookupSettingsPanel.indexButton.text=Index +HashLookupSettingsPanel.indexLabel.text=Index Status: +HashLookupModuleSettingsPanel.alwaysCalcHashesCheckbox.text=Calculate MD5 even if no hash database is selected +HashLookupModuleSettingsPanel.knownHashDbsLabel.text=Select known hash databases to use: +HashLookupModuleSettingsPanel.knownBadHashDbsLabel.text=Select known BAD hash databases to use: diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle_ja.properties b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle_ja.properties index 98df63128a..c6b81fcfa4 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle_ja.properties +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/Bundle_ja.properties @@ -23,31 +23,9 @@ HashDbImportDatabaseDialog.cancelButton.text=\u30ad\u30e3\u30f3\u30bb\u30eb HashDbCreateDatabaseDialog.jLabel2.text=\u7a2e\u985e\uff1a HashDbCreateDatabaseDialog.knownBadRadioButton.text=\u65e2\u77e5\u306e\u60aa\u8cea HashDbCreateDatabaseDialog.cancelButton.text=\u30ad\u30e3\u30f3\u30bb\u30eb -HashDbConfigPanel.nameLabel.text=\u30cf\u30c3\u30b7\u30e5\u30bb\u30c3\u30c8\u306e\u540d\u524d\uff1a -HashDbConfigPanel.hashDbNameLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 -HashDbConfigPanel.hashDatabasesLabel.text=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\uff1a -HashDbConfigPanel.hashDbLocationLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 -HashDbConfigPanel.ingestWarningLabel.text=\u51e6\u7406\u4e2d\u3067\u3059\u3002\u5b8c\u4e86\u3059\u308b\u307e\u3067\u4e00\u90e8\u306e\u8a2d\u5b9a\u306f\u5229\u7528\u3067\u304d\u307e\u305b\u3093\u3002 -HashDbConfigPanel.jButton3.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u30a4\u30f3\u30dd\u30fc\u30c8 -HashDbConfigPanel.jLabel6.text=\u7a2e\u985e\uff1a -HashDbConfigPanel.jLabel4.text=\u30ed\u30b1\u30fc\u30b7\u30e7\u30f3\uff1a -HashDbConfigPanel.jLabel2.text=\u540d\u524d\uff1a -HashDbConfigPanel.optionsLabel.text=\u30aa\u30d7\u30b7\u30e7\u30f3 -HashDbConfigPanel.typeLabel.text=\u7a2e\u985e\uff1a -HashDbConfigPanel.locationLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u30d1\u30b9\uff1a -HashDbConfigPanel.hashDbIndexStatusLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 -HashDbConfigPanel.hashDbTypeLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 -HashDbConfigPanel.indexButton.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9 -HashDbConfigPanel.indexLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u30b9\u30c6\u30fc\u30bf\u30b9\uff1a -HashDbConfigPanel.informationLabel.text=\u30a4\u30f3\u30d5\u30a9\u30e1\u30fc\u30b7\u30e7\u30f3 -HashDbConfigPanel.importDatabaseButton.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u30a4\u30f3\u30dd\u30fc\u30c8 -HashDbConfigPanel.deleteDatabaseButton.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u524a\u9664 -HashDbConfigPanel.indexPathLabelLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u30d1\u30b9\uff1a -HashDbConfigPanel.indexPathLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 ModalNoButtons.CURRENTLYON_LABEL.text=y\u306ex\u3092\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u5316\u4e2d ModalNoButtons.GO_GET_COFFEE_LABEL.text=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u5316\u4e2d\u3067\u3059\u3002\u6642\u9593\u304c\u304b\u304b\u308b\u5834\u5408\u304c\u3042\u308a\u307e\u3059\u3002 ModalNoButtons.CANCEL_BUTTON.text=\u30ad\u30e3\u30f3\u30bb\u30eb -HashDbSimpleConfigPanel.knownBadHashDbsLabel.text=\u65e2\u77e5\u306e\u60aa\u8cea\u306a\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u51e6\u7406\u306b\u5229\u7528\uff1a HashDbImportDatabaseDialog.knownRadioButton.text=\u65e2\u77e5\uff08NSRL\u307e\u305f\u306f\u305d\u306e\u4ed6\uff09 HashDbCreateDatabaseDialog.knownRadioButton.text=\u65e2\u77e5 HashDbCreateDatabaseDialog.jLabel1.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u30d1\u30b9\uff1a @@ -55,12 +33,9 @@ HashDbCreateDatabaseDialog.saveAsButton.text=\u540d\u524d\u3092\u3064\u3051\u306 HashDbImportDatabaseDialog.jLabel3.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u30d1\u30b9\uff1a HashDbCreateDatabaseDialog.sendIngestMessagesCheckbox.text=\u51e6\u7406\u30e1\u30c3\u30bb\u30fc\u30b8\u3092\u9001\u308b HashDbImportDatabaseDialog.sendIngestMessagesCheckbox.text=\u51e6\u7406\u30e1\u30c3\u30bb\u30fc\u30b8\u3092\u9001\u308b -HashDbConfigPanel.createDatabaseButton.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u4f5c\u6210 HashDbImportDatabaseDialog.openButton.text=\u958b\u304f... -HashDbSimpleConfigPanel.alwaysCalcHashesCheckbox.text=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u306a\u304f\u3066\u3082\u3001\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u8a08\u7b97 HashDbCreateDatabaseDialog.jLabel3.text=\u30cf\u30c3\u30b7\u30e5\u30bb\u30c3\u30c8\u306e\u540d\u524d\uff1a HashDbCreateDatabaseDialog.okButton.text=OK -HashDbConfigPanel.sendIngestMessagesCheckBox.text=\u51e6\u7406\u30e1\u30c3\u30bb\u30fc\u30b8\u3092\u9001\u308b AddContentToHashDbAction.ContentMenu.noHashDbsConfigd=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u8a2d\u5b9a\u3055\u308c\u3066\u3044\u307e\u305b\u3093 AddContentToHashDbAction.ContentMenu.createDbItem=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u4f5c\u6210... AddContentToHashDbAction.addFilesToHashSet.addToHashDbErr=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u30a8\u30e9\u30fc\u306b\u8ffd\u52a0 @@ -110,7 +85,6 @@ HashDbIngestModule.moduleDescription=\u6a19\u6e96\u306eNSRL\u30c7\u30fc\u30bf\u3 HashDbIngestModule.noKnownHashDbSetMsg=\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u5b58\u5728\u3057\u307e\u305b\u3093\u3002 HashDbIngestModule.knownFileSearchWillNotExecuteWarn=\u65e2\u77e5\u306e\u30d5\u30a1\u30a4\u30eb\u691c\u7d22\u304c\u5b9f\u884c\u3055\u308c\u307e\u305b\u3093\u3002 HashDbIngestModule.noKnownBadHashDbSetMsg=\u65e2\u77e5\u306e\u60aa\u8cea\u306a\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u30bb\u30c3\u30c8\u306f\u3042\u308a\u307e\u305b\u3093\u3002 -HashDbSimpleConfigPanel.knownHashDbsLabel.text=\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u51e6\u7406\u306b\u5229\u7528\uff1a HashDbConfigPanel.dbNotIndexedMsg=\u4e0b\u8a18\u306e\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u306f\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u5316\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u5316\u3057\u307e\u3059\u304b\uff1f\ {0} HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn=\u65e2\u77e5\u306e\u60aa\u8cea\u30d5\u30a1\u30a4\u30eb\u691c\u7d22\u306f\u5b9f\u884c\u3055\u308c\u307e\u305b\u3093\u3002 @@ -191,3 +165,29 @@ OptionsCategory_Name_HashDatabase=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30 OptionsCategory_Keywords_HashDatabase=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9 HashDbManager.ingestRunningExceptionMsg=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u4e2d\uff1b\u5b8c\u4e86\u3059\u308b\u307e\u3067\u3053\u306e\u30b5\u30fc\u30d3\u30b9\u306f\u5229\u7528\u3067\u304d\u307e\u305b\u3093\u3002 +HashLookupSettingsPanel.optionsLabel.text=\u30aa\u30d7\u30b7\u30e7\u30f3 +HashLookupSettingsPanel.jButton3.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u30a4\u30f3\u30dd\u30fc\u30c8 +HashLookupSettingsPanel.indexPathLabelLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u30d1\u30b9\uff1a +HashLookupSettingsPanel.createDatabaseButton.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u4f5c\u6210 +HashLookupSettingsPanel.jLabel6.text=\u7a2e\u985e\uff1a +HashLookupSettingsPanel.jLabel4.text=\u30ed\u30b1\u30fc\u30b7\u30e7\u30f3\uff1a +HashLookupSettingsPanel.jLabel2.text=\u540d\u524d\uff1a +HashLookupSettingsPanel.indexPathLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 +HashLookupSettingsPanel.ingestWarningLabel.text=\u51e6\u7406\u4e2d\u3067\u3059\u3002\u5b8c\u4e86\u3059\u308b\u307e\u3067\u4e00\u90e8\u306e\u8a2d\u5b9a\u306f\u5229\u7528\u3067\u304d\u307e\u305b\u3093\u3002 +HashLookupSettingsPanel.deleteDatabaseButton.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u524a\u9664 +HashLookupSettingsPanel.importDatabaseButton.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u30a4\u30f3\u30dd\u30fc\u30c8 +HashLookupSettingsPanel.hashDatabasesLabel.text=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\uff1a +HashLookupSettingsPanel.nameLabel.text=\u30cf\u30c3\u30b7\u30e5\u30bb\u30c3\u30c8\u306e\u540d\u524d\uff1a +HashLookupSettingsPanel.informationLabel.text=\u30a4\u30f3\u30d5\u30a9\u30e1\u30fc\u30b7\u30e7\u30f3 +HashLookupSettingsPanel.sendIngestMessagesCheckBox.text=\u51e6\u7406\u30e1\u30c3\u30bb\u30fc\u30b8\u3092\u9001\u308b +HashLookupSettingsPanel.hashDbLocationLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 +HashLookupSettingsPanel.hashDbNameLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 +HashLookupSettingsPanel.typeLabel.text=\u7a2e\u985e\uff1a +HashLookupSettingsPanel.locationLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u30d1\u30b9\uff1a +HashLookupSettingsPanel.hashDbIndexStatusLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 +HashLookupSettingsPanel.hashDbTypeLabel.text=\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u307e\u305b\u3093 +HashLookupSettingsPanel.indexButton.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9 +HashLookupSettingsPanel.indexLabel.text=\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u30b9\u30c6\u30fc\u30bf\u30b9\uff1a +HashLookupModuleSettingsPanel.alwaysCalcHashesCheckbox.text=\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u304c\u9078\u629e\u3055\u308c\u3066\u3044\u306a\u304f\u3066\u3082\u3001\u30cf\u30c3\u30b7\u30e5\u5024\u3092\u8a08\u7b97 +HashLookupModuleSettingsPanel.knownHashDbsLabel.text=\u65e2\u77e5\u306e\u30cf\u30c3\u30b7\u30e5\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u51e6\u7406\u306b\u5229\u7528\uff1a +HashLookupModuleSettingsPanel.knownBadHashDbsLabel.text=\u65e2\u77e5\u306e\u60aa\u8cea\u306a\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3092\u51e6\u7406\u306b\u5229\u7528\uff1a diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDatabaseOptionsPanelController.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDatabaseOptionsPanelController.java index 5ddac1c794..2fcc7ad52d 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDatabaseOptionsPanelController.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDatabaseOptionsPanelController.java @@ -41,7 +41,7 @@ id = "HashDatabase") //@org.openide.util.NbBundle.Messages({"OptionsCategory_Name_HashDatabase=Hash Database", "OptionsCategory_Keywords_HashDatabase=Hash Database"}) public final class HashDatabaseOptionsPanelController extends OptionsPanelController { - private HashDbConfigPanel panel; + private HashLookupSettingsPanel panel; private final PropertyChangeSupport pcs = new PropertyChangeSupport(this); private boolean changed; private static final Logger logger = Logger.getLogger(HashDatabaseOptionsPanelController.class.getName()); @@ -92,9 +92,9 @@ public final class HashDatabaseOptionsPanelController extends OptionsPanelContro pcs.removePropertyChangeListener(l); } - private HashDbConfigPanel getPanel() { + private HashLookupSettingsPanel getPanel() { if (panel == null) { - panel = new HashDbConfigPanel(); + panel = new HashLookupSettingsPanel(); } return panel; } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index 78bf96bae1..7d16a1c006 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -23,15 +23,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.logging.Level; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.Version; -import org.sleuthkit.autopsy.ingest.PipelineContext; import org.sleuthkit.autopsy.ingest.IngestMessage; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; -import org.sleuthkit.autopsy.ingest.IngestModuleInit; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; @@ -45,143 +40,61 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstract.IngestModuleException; +import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.datamodel.HashInfo; -public class HashDbIngestModule extends IngestModuleAbstractFile { - private static HashDbIngestModule instance = null; - public final static String MODULE_NAME = NbBundle.getMessage(HashDbIngestModule.class, - "HashDbIngestModule.moduleName"); - public final static String MODULE_DESCRIPTION = NbBundle.getMessage(HashDbIngestModule.class, - "HashDbIngestModule.moduleDescription"); - final public static String MODULE_VERSION = Version.getVersion(); +public class HashDbIngestModule extends IngestModuleAdapter implements FileIngestModule { private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName()); private static final int MAX_COMMENT_SIZE = 500; - private HashDbSimpleConfigPanel simpleConfigPanel; - private HashDbConfigPanel advancedConfigPanel; - private IngestServices services; - private SleuthkitCase skCase; - private static int messageId = 0; - private int knownBadCount = 0; - private boolean calcHashesIsSet; + private static int messageId = 0; // RJCTODO: This is not thread safe + private final IngestServices services = IngestServices.getDefault(); + private final Hash hasher = new Hash(); + private final SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); + private final HashDbManager hashDbManager = HashDbManager.getInstance(); + private final HashLookupModuleSettings settings; private List knownBadHashSets = new ArrayList<>(); private List knownHashSets = new ArrayList<>(); - static long calctime = 0; - static long lookuptime = 0; - private final Hash hasher = new Hash(); - - private HashDbIngestModule() { - } - - public static synchronized HashDbIngestModule getDefault() { - if (instance == null) { - instance = new HashDbIngestModule(); - } - return instance; - } - - @Override - public String getName() { - return MODULE_NAME; - } - - @Override - public String getDescription() { - return MODULE_DESCRIPTION; - } - - @Override - public String getVersion() { - return MODULE_VERSION; - } - - @Override - public boolean hasSimpleConfiguration() { - return true; - } + private int knownBadCount = 0; + private long calctime = 0; + private long lookuptime = 0; - @Override - public javax.swing.JPanel getSimpleConfiguration(String context) { - if (null == simpleConfigPanel) { - simpleConfigPanel = new HashDbSimpleConfigPanel(); - } - else { - simpleConfigPanel.load(); - } + HashDbIngestModule(HashLookupModuleSettings settings) { + this.settings = settings; + } - return simpleConfigPanel; - } - - @Override - public void saveSimpleConfiguration() { - if (simpleConfigPanel != null) { - simpleConfigPanel.store(); - } - } - - @Override - public boolean hasAdvancedConfiguration() { - return true; - } - - @Override - public javax.swing.JPanel getAdvancedConfiguration(String context) { - if (advancedConfigPanel == null) { - advancedConfigPanel = new HashDbConfigPanel(); - } - - advancedConfigPanel.load(); - return advancedConfigPanel; - } - - @Override - public void saveAdvancedConfiguration() { - if (advancedConfigPanel != null) { - advancedConfigPanel.store(); - } - - if (simpleConfigPanel != null) { - simpleConfigPanel.load(); - } - } - - @Override - public void init(IngestModuleInit initContext) throws IngestModuleException { - services = IngestServices.getDefault(); - skCase = Case.getCurrentCase().getSleuthkitCase(); - - HashDbManager hashDbManager = HashDbManager.getInstance(); - getHashSetsUsableForIngest(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); - getHashSetsUsableForIngest(hashDbManager.getKnownFileHashSets(), knownHashSets); - calcHashesIsSet = hashDbManager.getAlwaysCalculateHashes(); + @Override + public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException { + getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); + if (knownBadHashSets.isEmpty()) { + services.postMessage(IngestMessage.createWarningMessage(++messageId, + HashLookupModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.noKnownBadHashDbSetMsg"), + NbBundle.getMessage(this.getClass(), + "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn"))); + } + getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets); if (knownHashSets.isEmpty()) { services.postMessage(IngestMessage.createWarningMessage(++messageId, - this, + HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.noKnownHashDbSetMsg"), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.knownFileSearchWillNotExecuteWarn"))); } - if (knownBadHashSets.isEmpty()) { - services.postMessage(IngestMessage.createWarningMessage(++messageId, - this, - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.noKnownBadHashDbSetMsg"), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn"))); - } } - - private void getHashSetsUsableForIngest(List hashDbs, List hashDbsForIngest) { - assert hashDbs != null; - assert hashDbsForIngest != null; - hashDbsForIngest.clear(); - for (HashDb db : hashDbs) { - if (db.getSearchDuringIngest()) { + + private void getEnabledHashSets(List hashSets, List enabledHashSets) { + assert hashSets != null; + assert enabledHashSets != null; + enabledHashSets.clear(); + for (HashDb db : hashSets) { + if (settings.isHashSetEnabled(db.getHashSetName())) { try { if (db.hasIndex()) { - hashDbsForIngest.add(db); + enabledHashSets.add(db); } } catch (TskCoreException ex) { @@ -192,23 +105,14 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { } @Override - public boolean hasBackgroundJobsRunning() { - return false; - } - - @Override - public ProcessResult process(PipelineContextpipelineContext, AbstractFile file) { - //skip unalloc + public ProcessResult process(AbstractFile file) { + // Skip unallocated space files. if (file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { - return IngestModuleAbstractFile.ProcessResult.OK; + return ProcessResult.OK; } - - return processFile(file); - } - - private ProcessResult processFile(AbstractFile file) { + // bail out if we have no hashes set - if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (calcHashesIsSet == false)) { + if ((knownHashSets.isEmpty()) && (knownBadHashSets.isEmpty()) && (!settings.shouldCalculateHashes())) { return ProcessResult.OK; } @@ -223,7 +127,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { } catch (IOException ex) { logger.log(Level.WARNING, "Error calculating hash of file " + name, ex); services.postMessage(IngestMessage.createErrorMessage(++messageId, - HashDbIngestModule.this, + HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", name), @@ -249,7 +153,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { } catch (TskException ex) { logger.log(Level.WARNING, "Couldn't set known bad state for file " + name + " - see sleuthkit log for details", ex); services.postMessage(IngestMessage.createErrorMessage(++messageId, - HashDbIngestModule.this, + HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name), @@ -280,7 +184,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { } catch (TskException ex) { logger.log(Level.WARNING, "Couldn't lookup known bad hash for file " + name + " - see sleuthkit log for details", ex); services.postMessage(IngestMessage.createErrorMessage(++messageId, - HashDbIngestModule.this, + HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name), @@ -304,14 +208,6 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { break; } catch (TskException ex) { logger.log(Level.WARNING, "Couldn't set known state for file " + name + " - see sleuthkit log for details", ex); - services.postMessage(IngestMessage.createErrorMessage(++messageId, - HashDbIngestModule.this, - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.hashLookupErrorMsg", - name), - NbBundle.getMessage(this.getClass(), - "HashDbIngestModule.settingsKnownStateErr", - name))); ret = ProcessResult.ERROR; } } @@ -319,7 +215,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { } catch (TskException ex) { logger.log(Level.WARNING, "Couldn't lookup known hash for file " + name + " - see sleuthkit log for details", ex); services.postMessage(IngestMessage.createErrorMessage(++messageId, - HashDbIngestModule.this, + HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.hashLookupErrorMsg", name), @@ -333,9 +229,11 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { return ret; } - + private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) { try { + String MODULE_NAME = NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.moduleName"); + BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT); //TODO Revisit usage of deprecated constructor as per TSK-583 //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName); @@ -376,7 +274,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { detailsSb.append(""); - services.postMessage(IngestMessage.createDataMessage(++messageId, this, + services.postMessage(IngestMessage.createDataMessage(++messageId, HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.postToBB.knownBadMsg", abstractFile.getName()), @@ -392,7 +290,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { @Override - public void complete() { + public void shutDown(boolean ingestJobCancelled) { if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) { StringBuilder detailsSb = new StringBuilder(); //details @@ -421,14 +319,10 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { detailsSb.append(""); services.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, - this, + HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.hashLookupResults"), detailsSb.toString())); } } - - @Override - public void stop() { - } } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleFactory.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleFactory.java new file mode 100755 index 0000000000..92b7708524 --- /dev/null +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleFactory.java @@ -0,0 +1,118 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.hashdatabase; + +import java.util.ArrayList; +import java.util.List; +import org.openide.util.NbBundle; +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleFactory; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleSettingsPanel; +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel; + +/** + * A factory that creates file ingest modules that do hash database lookups. + */ +@ServiceProvider(service = IngestModuleFactory.class) +public class HashLookupModuleFactory extends IngestModuleFactoryAdapter { + + private HashLookupModuleSettingsPanel moduleSettingsPanel = null; + + @Override + public String getModuleDisplayName() { + return getModuleName(); + } + + static String getModuleName() { + return NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.moduleName"); + } + + @Override + public String getModuleDescription() { + return NbBundle.getMessage(HashDbIngestModule.class, "HashDbIngestModule.moduleDescription"); + } + + @Override + public String getModuleVersionNumber() { + return Version.getVersion(); + } + + @Override + public IngestModuleSettings getDefaultModuleSettings() { + HashDbManager hashDbManager = HashDbManager.getInstance(); + List enabledHashSets = new ArrayList<>(); + List knownFileHashSets = hashDbManager.getKnownFileHashSets(); + for (HashDbManager.HashDb db : knownFileHashSets) { + if (db.getSearchDuringIngest()) { + enabledHashSets.add(db.getHashSetName()); + } + } + List knownBadFileHashSets = hashDbManager.getKnownBadFileHashSets(); + for (HashDbManager.HashDb db : knownBadFileHashSets) { + if (db.getSearchDuringIngest()) { + enabledHashSets.add(db.getHashSetName()); + } + } + return new HashLookupModuleSettings(hashDbManager.getAlwaysCalculateHashes(), enabledHashSets); + } + + @Override + public boolean hasModuleSettingsPanel() { + return true; + } + + @Override + public IngestModuleSettingsPanel getModuleSettingsPanel(IngestModuleSettings settings) { + if (moduleSettingsPanel == null) { + moduleSettingsPanel = new HashLookupModuleSettingsPanel(); + } + moduleSettingsPanel.load(); // RJCTODO: Fix this, use passed in settings + return moduleSettingsPanel; + } + + @Override + public boolean hasGlobalSettingsPanel() { + return true; + } + + @Override + public IngestModuleGlobalSetttingsPanel getGlobalSettingsPanel() { + HashLookupSettingsPanel globalSettingsPanel = new HashLookupSettingsPanel(); + globalSettingsPanel.load(); + return globalSettingsPanel; + } + + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleSettings settings) { + assert settings instanceof HashLookupModuleSettings; + if (!(settings instanceof HashLookupModuleSettings)) { + throw new IllegalArgumentException("Expected settings argument to be instanceof HashLookupModuleSettings"); + } + return new HashDbIngestModule((HashLookupModuleSettings) settings); + } +} diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettings.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettings.java new file mode 100755 index 0000000000..c1671013ea --- /dev/null +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettings.java @@ -0,0 +1,47 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org * + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.hashdatabase; + +import java.util.HashSet; +import java.util.List; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; + +/** + * Settings for a hash lookup file ingest module instance. + */ +final class HashLookupModuleSettings implements IngestModuleSettings { + + private final HashSet enabledHashSets = new HashSet<>(); + private boolean shouldCalculateHashes = true; + + HashLookupModuleSettings(boolean shouldCalculateHashes, List enabledHashSetNames) { + this.shouldCalculateHashes = shouldCalculateHashes; + for (String hashSet : enabledHashSetNames) { + enabledHashSets.add(hashSet); + } + } + + boolean shouldCalculateHashes() { + return shouldCalculateHashes; + } + + boolean isHashSetEnabled(String hashSetName) { + return enabledHashSets.contains(hashSetName); + } +} diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSimpleConfigPanel.form b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.form similarity index 93% rename from HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSimpleConfigPanel.form rename to HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.form index a940de429e..dca060b4e9 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSimpleConfigPanel.form +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.form @@ -83,21 +83,21 @@ - + - + - + diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSimpleConfigPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.java similarity index 74% rename from HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSimpleConfigPanel.java rename to HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.java index 6ebe04ff03..e98fd281b9 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSimpleConfigPanel.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 - 2013 Basis Technology Corp. + * Copyright 2011 - 2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,11 +16,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.hashdatabase; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import javax.swing.JOptionPane; @@ -28,47 +28,41 @@ import javax.swing.JScrollPane; import javax.swing.JTable; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableColumn; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleSettingsPanel; /** - * Instances of this class provide a simplified UI for managing the hash sets configuration. + * Instances of this class provide a simplified UI for managing the hash sets + * configuration. */ -public class HashDbSimpleConfigPanel extends javax.swing.JPanel { - +public class HashLookupModuleSettingsPanel extends IngestModuleSettingsPanel implements PropertyChangeListener { + + private final HashDbManager hashDbManager = HashDbManager.getInstance(); private HashDatabasesTableModel knownTableModel; private HashDatabasesTableModel knownBadTableModel; - public HashDbSimpleConfigPanel() { + HashLookupModuleSettingsPanel() { knownTableModel = new HashDatabasesTableModel(HashDbManager.HashDb.KnownFilesType.KNOWN); knownBadTableModel = new HashDatabasesTableModel(HashDbManager.HashDb.KnownFilesType.KNOWN_BAD); initComponents(); customizeComponents(); } - + private void customizeComponents() { customizeHashDbsTable(jScrollPane1, knownHashTable, knownTableModel); customizeHashDbsTable(jScrollPane2, knownBadHashTable, knownBadTableModel); - alwaysCalcHashesCheckbox.setSelected(HashDbManager.getInstance().getAlwaysCalculateHashes()); - - // Add a listener to the always calculate hashes checkbox component. - // The listener passes the user's selection on to the hash database manager. - alwaysCalcHashesCheckbox.addActionListener( new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - HashDbManager.getInstance().setAlwaysCalculateHashes(alwaysCalcHashesCheckbox.isSelected()); - } - }); - + alwaysCalcHashesCheckbox.setSelected(hashDbManager.getAlwaysCalculateHashes()); load(); + hashDbManager.addPropertyChangeListener(this); } private void customizeHashDbsTable(JScrollPane scrollPane, JTable table, HashDatabasesTableModel tableModel) { - table.setModel(tableModel); + table.setModel(tableModel); table.setTableHeader(null); table.setRowSelectionAllowed(false); @@ -82,36 +76,62 @@ public class HashDbSimpleConfigPanel extends javax.swing.JPanel { } else { column.setPreferredWidth(((int) (width1 * 0.92))); } - } + } + } + + @Override + public void propertyChange(PropertyChangeEvent event) { + if (event.getPropertyName().equals(HashDbManager.SetEvt.DB_ADDED.name()) || + event.getPropertyName().equals(HashDbManager.SetEvt.DB_DELETED.name())) { + load(); + } } - public void load() { + @Override + public IngestModuleSettings getSettings() { + List enabledHashSets = new ArrayList<>(); + List knownFileHashSets = hashDbManager.getKnownFileHashSets(); + for (HashDb db : knownFileHashSets) { + if (db.getSearchDuringIngest()) { + enabledHashSets.add(db.getHashSetName()); + } + } + List knownBadFileHashSets = hashDbManager.getKnownBadFileHashSets(); + for (HashDb db : knownBadFileHashSets) { + if (db.getSearchDuringIngest()) { + enabledHashSets.add(db.getHashSetName()); + } + } + return new HashLookupModuleSettings(alwaysCalcHashesCheckbox.isSelected(), enabledHashSets); + } + + void load() { knownTableModel.load(); knownBadTableModel.load(); } - public void store() { - HashDbManager.getInstance().save(); + void store() { + hashDbManager.save(); } - - private class HashDatabasesTableModel extends AbstractTableModel { - private final HashDbManager.HashDb.KnownFilesType hashDatabasesType; + + private class HashDatabasesTableModel extends AbstractTableModel { + + private final HashDbManager.HashDb.KnownFilesType hashDatabasesType; private List hashDatabases; - + HashDatabasesTableModel(HashDbManager.HashDb.KnownFilesType hashDatabasesType) { this.hashDatabasesType = hashDatabasesType; getHashDatabases(); } - + private void getHashDatabases() { if (HashDbManager.HashDb.KnownFilesType.KNOWN == hashDatabasesType) { - hashDatabases = HashDbManager.getInstance().getKnownFileHashSets(); + hashDatabases = hashDbManager.getKnownFileHashSets(); + } else { + hashDatabases = hashDbManager.getKnownBadFileHashSets(); } - else { - hashDatabases = HashDbManager.getInstance().getKnownBadFileHashSets(); - } - } - + } + private void load() { getHashDatabases(); fireTableDataChanged(); @@ -136,7 +156,7 @@ public class HashDbSimpleConfigPanel extends javax.swing.JPanel { return db.getHashSetName(); } } - + @Override public boolean isCellEditable(int rowIndex, int columnIndex) { return !IngestManager.getDefault().isIngestRunning() && columnIndex == 0; @@ -144,36 +164,34 @@ public class HashDbSimpleConfigPanel extends javax.swing.JPanel { @Override public void setValueAt(Object aValue, int rowIndex, int columnIndex) { - if(columnIndex == 0) { + if (columnIndex == 0) { HashDb db = hashDatabases.get(rowIndex); boolean dbHasIndex = false; try { dbHasIndex = db.hasIndex(); + } catch (TskCoreException ex) { + Logger.getLogger(HashLookupModuleSettingsPanel.class.getName()).log(Level.SEVERE, "Error getting info for " + db.getHashSetName() + " hash database", ex); } - catch (TskCoreException ex) { - Logger.getLogger(HashDbSimpleConfigPanel.class.getName()).log(Level.SEVERE, "Error getting info for " + db.getHashSetName() + " hash database", ex); - } - if(((Boolean) getValueAt(rowIndex, columnIndex)) || dbHasIndex) { + if (((Boolean) getValueAt(rowIndex, columnIndex)) || dbHasIndex) { db.setSearchDuringIngest((Boolean) aValue); - } - else { - JOptionPane.showMessageDialog(HashDbSimpleConfigPanel.this, - NbBundle.getMessage(this.getClass(), - "HashDbSimpleConfigPanel.dlgMsg.mustIndexDbBeforeUse")); + } else { + JOptionPane.showMessageDialog(HashLookupModuleSettingsPanel.this, + NbBundle.getMessage(this.getClass(), + "HashDbSimpleConfigPanel.dlgMsg.mustIndexDbBeforeUse")); } } } - + @Override public Class getColumnClass(int c) { return getValueAt(0, c).getClass(); } } - - /** This method is called from within the constructor to - * initialize the form. - * WARNING: Do NOT modify this code. The content of this method is - * always regenerated by the Form Editor. + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. */ @SuppressWarnings("unchecked") // //GEN-BEGIN:initComponents @@ -194,11 +212,11 @@ public class HashDbSimpleConfigPanel extends javax.swing.JPanel { knownHashTable.setShowVerticalLines(false); jScrollPane1.setViewportView(knownHashTable); - knownBadHashDbsLabel.setText(org.openide.util.NbBundle.getMessage(HashDbSimpleConfigPanel.class, "HashDbSimpleConfigPanel.knownBadHashDbsLabel.text")); // NOI18N + knownBadHashDbsLabel.setText(org.openide.util.NbBundle.getMessage(HashLookupModuleSettingsPanel.class, "HashLookupModuleSettingsPanel.knownBadHashDbsLabel.text")); // NOI18N - knownHashDbsLabel.setText(org.openide.util.NbBundle.getMessage(HashDbSimpleConfigPanel.class, "HashDbSimpleConfigPanel.knownHashDbsLabel.text")); // NOI18N + knownHashDbsLabel.setText(org.openide.util.NbBundle.getMessage(HashLookupModuleSettingsPanel.class, "HashLookupModuleSettingsPanel.knownHashDbsLabel.text")); // NOI18N - alwaysCalcHashesCheckbox.setText(org.openide.util.NbBundle.getMessage(HashDbSimpleConfigPanel.class, "HashDbSimpleConfigPanel.alwaysCalcHashesCheckbox.text")); // NOI18N + alwaysCalcHashesCheckbox.setText(org.openide.util.NbBundle.getMessage(HashLookupModuleSettingsPanel.class, "HashLookupModuleSettingsPanel.alwaysCalcHashesCheckbox.text")); // NOI18N alwaysCalcHashesCheckbox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { alwaysCalcHashesCheckboxActionPerformed(evt); @@ -256,9 +274,8 @@ public class HashDbSimpleConfigPanel extends javax.swing.JPanel { }// //GEN-END:initComponents private void alwaysCalcHashesCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_alwaysCalcHashesCheckboxActionPerformed - // TODO add your handling code here: + hashDbManager.setAlwaysCalculateHashes(alwaysCalcHashesCheckbox.isSelected()); }//GEN-LAST:event_alwaysCalcHashesCheckboxActionPerformed - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JCheckBox alwaysCalcHashesCheckbox; private javax.swing.JScrollPane jScrollPane1; diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbConfigPanel.form b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.form similarity index 89% rename from HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbConfigPanel.form rename to HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.form index d08cc26690..da7ad7a545 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbConfigPanel.form +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.form @@ -5,21 +5,21 @@ - + - + - + @@ -29,7 +29,7 @@ - + @@ -206,7 +206,7 @@ - + @@ -246,7 +246,7 @@ - + @@ -268,7 +268,7 @@ - + @@ -287,70 +287,70 @@ - + - + - + - + - + - + - + - + - + - + @@ -361,7 +361,7 @@ - + @@ -371,14 +371,14 @@ - + - + @@ -392,7 +392,7 @@ - + @@ -411,14 +411,14 @@ - + - + diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbConfigPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java similarity index 81% rename from HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbConfigPanel.java rename to HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java index 6393f44452..c84dd442b1 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbConfigPanel.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java @@ -44,25 +44,28 @@ import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb.KnownFilesType; +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel; /** - * Instances of this class provide a comprehensive UI for managing the hash sets configuration. + * Instances of this class provide a comprehensive UI for managing the hash sets + * configuration. */ -public final class HashDbConfigPanel extends javax.swing.JPanel implements OptionsPanel { +public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPanel implements OptionsPanel { + private static final String NO_SELECTION_TEXT = NbBundle - .getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.noSelectionText"); + .getMessage(HashLookupSettingsPanel.class, "HashDbConfigPanel.noSelectionText"); private static final String ERROR_GETTING_PATH_TEXT = NbBundle - .getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.errorGettingPathText"); + .getMessage(HashLookupSettingsPanel.class, "HashDbConfigPanel.errorGettingPathText"); private static final String ERROR_GETTING_INDEX_STATUS_TEXT = NbBundle - .getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.errorGettingIndexStatusText"); + .getMessage(HashLookupSettingsPanel.class, "HashDbConfigPanel.errorGettingIndexStatusText"); private HashDbManager hashSetManager = HashDbManager.getInstance(); - private HashSetTableModel hashSetTableModel = new HashSetTableModel(); - - public HashDbConfigPanel() { + private HashSetTableModel hashSetTableModel = new HashSetTableModel(); + + public HashLookupSettingsPanel() { initComponents(); customizeComponents(); updateComponentsForNoSelection(); - + // Listen to the ingest modules to refresh the enabled/disabled state of // the components in sync with file ingest. IngestManager.addPropertyChangeListener(new PropertyChangeListener() { @@ -72,9 +75,9 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio updateComponents(); } } - }); + }); } - + private void customizeComponents() { setName(NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.setName.hashSetConfig")); this.ingestWarningLabel.setVisible(false); @@ -89,21 +92,20 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio updateComponents(); } } - }); + }); } private void updateComponents() { - HashDb db = ((HashSetTable)hashSetTable).getSelection(); + HashDb db = ((HashSetTable) hashSetTable).getSelection(); if (db != null) { updateComponentsForSelection(db); - } - else { + } else { updateComponentsForNoSelection(); - } + } } - + private void updateComponentsForNoSelection() { - boolean ingestIsRunning = IngestManager.getDefault().isIngestRunning(); + boolean ingestIsRunning = IngestManager.getDefault().isIngestRunning(); // Update descriptive labels. hashDbNameLabel.setText(NO_SELECTION_TEXT); @@ -115,46 +117,44 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio hashDbIndexStatusLabel.setText(NO_SELECTION_TEXT); hashDbIndexStatusLabel.setForeground(Color.black); indexButton.setText(NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexButtonText.index")); - indexButton.setEnabled(false); + indexButton.setEnabled(false); // Update ingest options. sendIngestMessagesCheckBox.setSelected(false); sendIngestMessagesCheckBox.setEnabled(false); optionsLabel.setEnabled(false); optionsSeparator.setEnabled(false); - + // Update database action buttons. createDatabaseButton.setEnabled(true); importDatabaseButton.setEnabled(true); deleteDatabaseButton.setEnabled(false); - + // Update ingest in progress warning label. - ingestWarningLabel.setVisible(ingestIsRunning); + ingestWarningLabel.setVisible(ingestIsRunning); } - private void updateComponentsForSelection(HashDb db) { - boolean ingestIsRunning = IngestManager.getDefault().isIngestRunning(); + private void updateComponentsForSelection(HashDb db) { + boolean ingestIsRunning = IngestManager.getDefault().isIngestRunning(); // Update descriptive labels. hashDbNameLabel.setText(db.getHashSetName()); hashDbTypeLabel.setText(db.getKnownFilesType().getDisplayName()); - + try { hashDbLocationLabel.setText(shortenPath(db.getDatabasePath())); - } - catch (TskCoreException ex) { - Logger.getLogger(HashDbConfigPanel.class.getName()).log(Level.SEVERE, "Error getting database path of " + db.getHashSetName() + " hash database", ex); + } catch (TskCoreException ex) { + Logger.getLogger(HashLookupSettingsPanel.class.getName()).log(Level.SEVERE, "Error getting database path of " + db.getHashSetName() + " hash database", ex); hashDbLocationLabel.setText(ERROR_GETTING_PATH_TEXT); } - + try { indexPathLabel.setText(shortenPath(db.getIndexPath())); - } - catch (TskCoreException ex) { - Logger.getLogger(HashDbConfigPanel.class.getName()).log(Level.SEVERE, "Error getting index path of " + db.getHashSetName() + " hash database", ex); + } catch (TskCoreException ex) { + Logger.getLogger(HashLookupSettingsPanel.class.getName()).log(Level.SEVERE, "Error getting index path of " + db.getHashSetName() + " hash database", ex); indexPathLabel.setText(ERROR_GETTING_PATH_TEXT); } - + // Update indexing components. try { if (db.isIndexing()) { @@ -164,13 +164,11 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexStatusText.indexGen")); hashDbIndexStatusLabel.setForeground(Color.black); indexButton.setEnabled(false); - } - else if (db.hasIndex()) { + } else if (db.hasIndex()) { if (db.hasIndexOnly()) { hashDbIndexStatusLabel.setText( NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexStatusText.indexOnly")); - } - else { + } else { hashDbIndexStatusLabel.setText( NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexStatusText.indexed")); } @@ -179,28 +177,25 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio indexButton.setText( NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexButtonText.reIndex")); indexButton.setEnabled(true); - } - else { + } else { indexButton.setText(NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexButtonText.index")); - indexButton.setEnabled(false); + indexButton.setEnabled(false); } - } - else { + } else { hashDbIndexStatusLabel.setText( NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexStatusText.noIndex")); hashDbIndexStatusLabel.setForeground(Color.red); indexButton.setText(NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexButtonText.index")); indexButton.setEnabled(true); } - } - catch (TskCoreException ex) { - Logger.getLogger(HashDbConfigPanel.class.getName()).log(Level.SEVERE, "Error getting index state of hash database", ex); + } catch (TskCoreException ex) { + Logger.getLogger(HashLookupSettingsPanel.class.getName()).log(Level.SEVERE, "Error getting index state of hash database", ex); hashDbIndexStatusLabel.setText(ERROR_GETTING_INDEX_STATUS_TEXT); hashDbIndexStatusLabel.setForeground(Color.red); indexButton.setText(NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.indexButtonText.index")); indexButton.setEnabled(false); - } - + } + // Disable the indexing button if ingest is in progress. if (ingestIsRunning) { indexButton.setEnabled(false); @@ -211,36 +206,30 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio sendIngestMessagesCheckBox.setEnabled(!ingestIsRunning && db.getSearchDuringIngest() && db.getKnownFilesType().equals(KnownFilesType.KNOWN_BAD)); optionsLabel.setEnabled(!ingestIsRunning); optionsSeparator.setEnabled(!ingestIsRunning); - + // Update database action buttons. createDatabaseButton.setEnabled(true); importDatabaseButton.setEnabled(true); deleteDatabaseButton.setEnabled(!ingestIsRunning); - + // Update ingest in progress warning label. - ingestWarningLabel.setVisible(ingestIsRunning); + ingestWarningLabel.setVisible(ingestIsRunning); } - + private static String shortenPath(String path) { String shortenedPath = path; - if (shortenedPath.length() > 50){ + if (shortenedPath.length() > 50) { shortenedPath = shortenedPath.substring(0, 10 + shortenedPath.substring(10).indexOf(File.separator) + 1) + "..." + shortenedPath.substring((shortenedPath.length() - 20) + shortenedPath.substring(shortenedPath.length() - 20).indexOf(File.separator)); } return shortenedPath; } - - private boolean isFileIngestStatusChangeEvent(PropertyChangeEvent evt) { - return evt.getPropertyName().equals(IngestManager.IngestModuleEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestModuleEvent.COMPLETED.toString()) || evt.getPropertyName().equals(IngestManager.IngestModuleEvent.STOPPED.toString()); - } - - @Override - public void load() { - hashSetTable.clearSelection(); - hashSetTableModel.refreshModel(); + + private boolean isFileIngestStatusChangeEvent(PropertyChangeEvent evt) { + return evt.getPropertyName().equals(IngestManager.IngestEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.COMPLETED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.STOPPED.toString()); } @Override - public void store() { + public void saveSettings() { //Checking for for any unindexed databases List unindexed = new ArrayList<>(); for (HashDb hashSet : hashSetManager.getAllHashSets()) { @@ -248,66 +237,76 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio if (!hashSet.hasIndex()) { unindexed.add(hashSet); } - } - catch (TskCoreException ex) { - Logger.getLogger(HashDbConfigPanel.class.getName()).log(Level.SEVERE, "Error getting index info for hash database", ex); + } catch (TskCoreException ex) { + Logger.getLogger(HashLookupSettingsPanel.class.getName()).log(Level.SEVERE, "Error getting index info for hash database", ex); } } //If unindexed ones are found, show a popup box that will either index them, or remove them. - if (unindexed.size() == 1){ + if (unindexed.size() == 1) { showInvalidIndex(false, unindexed); - } - else if (unindexed.size() > 1){ + } else if (unindexed.size() > 1) { showInvalidIndex(true, unindexed); } - hashSetManager.save(); + hashSetManager.save(); } - + + @Override + public void load() { + hashSetTable.clearSelection(); + hashSetTableModel.refreshModel(); + } + + @Override + public void store() { + saveSettings(); + } + public void cancel() { - HashDbManager.getInstance().loadLastSavedConfiguration(); + HashDbManager.getInstance().loadLastSavedConfiguration(); } - + void removeThese(List toRemove) { for (HashDb hashDb : toRemove) { hashSetManager.removeHashDatabaseInternal(hashDb); } - hashSetTableModel.refreshModel(); + hashSetTableModel.refreshModel(); } - + /** - * Displays the popup box that tells user that some of his databases are unindexed, along with solutions. - * This method is related to ModalNoButtons, to be removed at a later date. + * Displays the popup box that tells user that some of his databases are + * unindexed, along with solutions. This method is related to + * ModalNoButtons, to be removed at a later date. + * * @param plural Whether or not there are multiple unindexed databases * @param unindexed The list of unindexed databases. Can be of size 1. */ - private void showInvalidIndex(boolean plural, List unindexed){ + private void showInvalidIndex(boolean plural, List unindexed) { String total = ""; String message; - for(HashDb hdb : unindexed){ - total+= "\n" + hdb.getHashSetName(); + for (HashDb hdb : unindexed) { + total += "\n" + hdb.getHashSetName(); } - if(plural){ + if (plural) { message = NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.dbsNotIndexedMsg", total); - } - else{ + } else { message = NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.dbNotIndexedMsg", total); } int res = JOptionPane.showConfirmDialog(this, message, - NbBundle.getMessage(this.getClass(), - "HashDbConfigPanel.unindexedDbsMsg"), - JOptionPane.YES_NO_OPTION); - if(res == JOptionPane.YES_OPTION){ - ModalNoButtons indexingDialog = new ModalNoButtons(this, new Frame(),unindexed); + NbBundle.getMessage(this.getClass(), + "HashDbConfigPanel.unindexedDbsMsg"), + JOptionPane.YES_NO_OPTION); + if (res == JOptionPane.YES_OPTION) { + ModalNoButtons indexingDialog = new ModalNoButtons(this, new Frame(), unindexed); indexingDialog.setLocationRelativeTo(null); indexingDialog.setVisible(true); indexingDialog.setModal(true); hashSetTableModel.refreshModel(); } - if(res == JOptionPane.NO_OPTION){ + if (res == JOptionPane.NO_OPTION) { JOptionPane.showMessageDialog(this, NbBundle.getMessage(this.getClass(), - "HashDbConfigPanel.allUnindexedDbsRmFromListMsg")); + "HashDbConfigPanel.allUnindexedDbsRmFromListMsg")); removeThese(unindexed); } } @@ -315,30 +314,30 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio boolean valid() { return true; } - - /** + + /** * This class implements a table for displaying configured hash sets. - */ + */ private class HashSetTable extends JTable { + @Override public Component prepareRenderer(TableCellRenderer renderer, int row, int column) { // Use the hash set name as the cell text. - JComponent cellRenderer = (JComponent)super.prepareRenderer(renderer, row, column); - cellRenderer.setToolTipText((String)getValueAt(row, column)); - + JComponent cellRenderer = (JComponent) super.prepareRenderer(renderer, row, column); + cellRenderer.setToolTipText((String) getValueAt(row, column)); + // Give the user a visual indication of any hash sets with a hash // database that needs to be indexed by displaying the hash set name // in red. - if (hashSetTableModel.indexExists(row)){ + if (hashSetTableModel.indexExists(row)) { cellRenderer.setForeground(Color.black); - } - else{ + } else { cellRenderer.setForeground(Color.red); } - + return cellRenderer; } - + public HashDb getSelection() { return hashSetTableModel.getHashSetAt(getSelectionModel().getMinSelectionIndex()); } @@ -348,19 +347,20 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio getSelectionModel().setSelectionInterval(index, index); } } - + public void selectRowByName(String name) { setSelection(hashSetTableModel.getIndexByName(name)); - } + } } - + /** * This class implements the table model for the table used to display * configured hash sets. - */ - private class HashSetTableModel extends AbstractTableModel { + */ + private class HashSetTableModel extends AbstractTableModel { + List hashSets = HashDbManager.getInstance().getAllHashSets(); - + @Override public int getColumnCount() { return 1; @@ -380,17 +380,16 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio public Object getValueAt(int rowIndex, int columnIndex) { return hashSets.get(rowIndex).getHashSetName(); } - - private boolean indexExists(int rowIndex){ + + private boolean indexExists(int rowIndex) { try { return hashSets.get(rowIndex).hasIndex(); - } - catch (TskCoreException ex) { - Logger.getLogger(HashSetTableModel.class.getName()).log(Level.SEVERE, "Error getting index info for hash database", ex); + } catch (TskCoreException ex) { + Logger.getLogger(HashSetTableModel.class.getName()).log(Level.SEVERE, "Error getting index info for hash database", ex); return false; } } - + @Override public boolean isCellEditable(int rowIndex, int columnIndex) { return false; @@ -409,32 +408,31 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio HashDb getHashSetAt(int index) { if (!hashSets.isEmpty() && index >= 0 && index < hashSets.size()) { - return hashSets.get(index); - } - else { + return hashSets.get(index); + } else { return null; } } - + int getIndexByName(String name) { for (int i = 0; i < hashSets.size(); ++i) { if (hashSets.get(i).getHashSetName().equals(name)) { - return i; - } - } + return i; + } + } return -1; } - + void refreshModel() { hashSets = HashDbManager.getInstance().getAllHashSets(); refreshDisplay(); } - + void refreshDisplay() { - fireTableDataChanged(); + fireTableDataChanged(); } } - + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -471,20 +469,20 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio indexPathLabelLabel = new javax.swing.JLabel(); indexPathLabel = new javax.swing.JLabel(); - org.openide.awt.Mnemonics.setLocalizedText(jLabel2, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.jLabel2.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(jLabel2, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.jLabel2.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.jLabel4.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.jLabel4.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(jLabel6, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.jLabel6.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(jLabel6, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.jLabel6.text")); // NOI18N jButton3.setFont(new java.awt.Font("Tahoma", 0, 14)); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(jButton3, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.jButton3.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(jButton3, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.jButton3.text")); // NOI18N setMinimumSize(new java.awt.Dimension(700, 500)); setPreferredSize(new java.awt.Dimension(700, 500)); ingestWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/hashdatabase/warning16.png"))); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(ingestWarningLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.ingestWarningLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(ingestWarningLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.ingestWarningLabel.text")); // NOI18N hashSetTable.setModel(new javax.swing.table.DefaultTableModel( new Object [][] { @@ -504,7 +502,7 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio jScrollPane1.setViewportView(hashSetTable); deleteDatabaseButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/hashdatabase/delete16.png"))); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(deleteDatabaseButton, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.deleteDatabaseButton.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(deleteDatabaseButton, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.deleteDatabaseButton.text")); // NOI18N deleteDatabaseButton.setMaximumSize(new java.awt.Dimension(140, 25)); deleteDatabaseButton.setMinimumSize(new java.awt.Dimension(140, 25)); deleteDatabaseButton.setPreferredSize(new java.awt.Dimension(140, 25)); @@ -515,7 +513,7 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio }); importDatabaseButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/hashdatabase/import16.png"))); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(importDatabaseButton, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.importDatabaseButton.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(importDatabaseButton, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.importDatabaseButton.text")); // NOI18N importDatabaseButton.setMaximumSize(new java.awt.Dimension(140, 25)); importDatabaseButton.setMinimumSize(new java.awt.Dimension(140, 25)); importDatabaseButton.setPreferredSize(new java.awt.Dimension(140, 25)); @@ -525,25 +523,25 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio } }); - org.openide.awt.Mnemonics.setLocalizedText(hashDatabasesLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.hashDatabasesLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(hashDatabasesLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.hashDatabasesLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(nameLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.nameLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(nameLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.nameLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(hashDbNameLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.hashDbNameLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(hashDbNameLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.hashDbNameLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(hashDbLocationLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.hashDbLocationLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(hashDbLocationLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.hashDbLocationLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(locationLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.locationLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(locationLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.locationLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(typeLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.typeLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(typeLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.typeLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(hashDbTypeLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.hashDbTypeLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(hashDbTypeLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.hashDbTypeLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(hashDbIndexStatusLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.hashDbIndexStatusLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(hashDbIndexStatusLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.hashDbIndexStatusLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(indexLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.indexLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(indexLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.indexLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(indexButton, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.indexButton.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(indexButton, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.indexButton.text")); // NOI18N indexButton.setEnabled(false); indexButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -551,19 +549,19 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio } }); - org.openide.awt.Mnemonics.setLocalizedText(sendIngestMessagesCheckBox, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.sendIngestMessagesCheckBox.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(sendIngestMessagesCheckBox, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.sendIngestMessagesCheckBox.text")); // NOI18N sendIngestMessagesCheckBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { sendIngestMessagesCheckBoxActionPerformed(evt); } }); - org.openide.awt.Mnemonics.setLocalizedText(informationLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.informationLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(informationLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.informationLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(optionsLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.optionsLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(optionsLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.optionsLabel.text")); // NOI18N createDatabaseButton.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/hashdatabase/new16.png"))); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(createDatabaseButton, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.createDatabaseButton.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(createDatabaseButton, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.createDatabaseButton.text")); // NOI18N createDatabaseButton.setMaximumSize(new java.awt.Dimension(140, 25)); createDatabaseButton.setMinimumSize(new java.awt.Dimension(140, 25)); createDatabaseButton.setPreferredSize(new java.awt.Dimension(140, 25)); @@ -573,9 +571,9 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio } }); - org.openide.awt.Mnemonics.setLocalizedText(indexPathLabelLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.indexPathLabelLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(indexPathLabelLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.indexPathLabelLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(indexPathLabel, org.openide.util.NbBundle.getMessage(HashDbConfigPanel.class, "HashDbConfigPanel.indexPathLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(indexPathLabel, org.openide.util.NbBundle.getMessage(HashLookupSettingsPanel.class, "HashLookupSettingsPanel.indexPathLabel.text")); // NOI18N javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -692,22 +690,22 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio }// //GEN-END:initComponents private void indexButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_indexButtonActionPerformed - final HashDb hashDb = ((HashSetTable)hashSetTable).getSelection(); - assert hashDb != null; - + final HashDb hashDb = ((HashSetTable) hashSetTable).getSelection(); + assert hashDb != null; + // Add a listener for the INDEXING_DONE event. This listener will update // the UI. hashDb.addPropertyChangeListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { if (evt.getPropertyName().equals(HashDb.Event.INDEXING_DONE.toString())) { - HashDb selectedHashDb = ((HashSetTable)hashSetTable).getSelection(); + HashDb selectedHashDb = ((HashSetTable) hashSetTable).getSelection(); if (selectedHashDb != null && hashDb != null && hashDb.equals(selectedHashDb)) { updateComponents(); } hashSetTableModel.refreshDisplay(); } - } + } }); // Display a modal dialog box to kick off the indexing on a worker thread @@ -718,17 +716,17 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio ModalNoButtons indexDialog = new ModalNoButtons(this, new Frame(), hashDb); indexDialog.setLocationRelativeTo(null); indexDialog.setVisible(true); - indexDialog.setModal(true); + indexDialog.setModal(true); }//GEN-LAST:event_indexButtonActionPerformed private void deleteDatabaseButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteDatabaseButtonActionPerformed if (JOptionPane.showConfirmDialog(null, - NbBundle.getMessage(this.getClass(), - "HashDbConfigPanel.deleteDbActionConfirmMsg"), - NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.deleteDbActionMsg"), - JOptionPane.YES_NO_OPTION, - JOptionPane.WARNING_MESSAGE) == JOptionPane.YES_OPTION) { - HashDb hashDb = ((HashSetTable)hashSetTable).getSelection(); + NbBundle.getMessage(this.getClass(), + "HashDbConfigPanel.deleteDbActionConfirmMsg"), + NbBundle.getMessage(this.getClass(), "HashDbConfigPanel.deleteDbActionMsg"), + JOptionPane.YES_NO_OPTION, + JOptionPane.WARNING_MESSAGE) == JOptionPane.YES_OPTION) { + HashDb hashDb = ((HashSetTable) hashSetTable).getSelection(); if (hashDb != null) { hashSetManager.removeHashDatabaseInternal(hashDb); hashSetTableModel.refreshModel(); @@ -738,16 +736,16 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio private void hashSetTableKeyPressed(java.awt.event.KeyEvent evt) {//GEN-FIRST:event_hashSetTableKeyPressed if (evt.getKeyCode() == KeyEvent.VK_DELETE) { - HashDb hashDb = ((HashSetTable)hashSetTable).getSelection(); + HashDb hashDb = ((HashSetTable) hashSetTable).getSelection(); if (hashDb != null) { hashSetManager.removeHashDatabaseInternal(hashDb); hashSetTableModel.refreshModel(); } - } + } }//GEN-LAST:event_hashSetTableKeyPressed private void sendIngestMessagesCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_sendIngestMessagesCheckBoxActionPerformed - HashDb hashDb = ((HashSetTable)hashSetTable).getSelection(); + HashDb hashDb = ((HashSetTable) hashSetTable).getSelection(); if (hashDb != null) { hashDb.setSendIngestMessages(sendIngestMessagesCheckBox.isSelected()); } @@ -757,18 +755,17 @@ public final class HashDbConfigPanel extends javax.swing.JPanel implements Optio HashDb hashDb = new HashDbImportDatabaseDialog().getHashDatabase(); if (null != hashDb) { hashSetTableModel.refreshModel(); - ((HashSetTable)hashSetTable).selectRowByName(hashDb.getHashSetName()); - } + ((HashSetTable) hashSetTable).selectRowByName(hashDb.getHashSetName()); + } }//GEN-LAST:event_importDatabaseButtonActionPerformed private void createDatabaseButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_createDatabaseButtonActionPerformed HashDb hashDb = new HashDbCreateDatabaseDialog().getHashDatabase(); if (null != hashDb) { hashSetTableModel.refreshModel(); - ((HashSetTable)hashSetTable).selectRowByName(hashDb.getHashSetName()); + ((HashSetTable) hashSetTable).selectRowByName(hashDb.getHashSetName()); } }//GEN-LAST:event_createDatabaseButtonActionPerformed - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton createDatabaseButton; private javax.swing.JButton deleteDatabaseButton; diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/ModalNoButtons.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/ModalNoButtons.java index 58cc01b58d..d2b2cda389 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/ModalNoButtons.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/ModalNoButtons.java @@ -42,7 +42,7 @@ class ModalNoButtons extends javax.swing.JDialog implements PropertyChangeListen List unindexed; HashDb toIndex; - HashDbConfigPanel hdbmp; + HashLookupSettingsPanel hdbmp; int length = 0; int currentcount = 1; String currentDb = ""; @@ -53,7 +53,7 @@ class ModalNoButtons extends javax.swing.JDialog implements PropertyChangeListen * @param parent Swing parent frame. * @param unindexed the list of unindexed databases to index. */ - ModalNoButtons(HashDbConfigPanel hdbmp, java.awt.Frame parent, List unindexed) { + ModalNoButtons(HashLookupSettingsPanel hdbmp, java.awt.Frame parent, List unindexed) { super(parent, NbBundle.getMessage(ModalNoButtons.class, "ModalNoButtons.indexingDbsTitle"), true); this.unindexed = unindexed; this.toIndex = null; @@ -68,7 +68,7 @@ class ModalNoButtons extends javax.swing.JDialog implements PropertyChangeListen * @param parent Swing parent frame. * @param unindexed The unindexed database to index. */ - ModalNoButtons(HashDbConfigPanel hdbmp, java.awt.Frame parent, HashDb unindexed){ + ModalNoButtons(HashLookupSettingsPanel hdbmp, java.awt.Frame parent, HashDb unindexed){ super(parent, NbBundle.getMessage(ModalNoButtons.class, "ModalNoButtons.indexingDbTitle"), true); this.unindexed = null; this.toIndex = unindexed; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java index 3255df7593..e013ec2fac 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java @@ -61,8 +61,8 @@ import org.sleuthkit.datamodel.ReadContentInputStream; //"application/xml-dtd", ); - AbstractFileHtmlExtract() { - this.module = KeywordSearchIngestModule.getDefault(); + AbstractFileHtmlExtract(KeywordSearchIngestModule module) { + this.module = module; ingester = Server.getIngester(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java index 15c6aac52c..4d82827c6a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,8 +60,8 @@ class AbstractFileStringExtract implements AbstractFileExtract { //STRING_CHUNK_BUF[1] = (byte) 0xBB; //STRING_CHUNK_BUF[2] = (byte) 0xBF; //} - public AbstractFileStringExtract() { - this.module = KeywordSearchIngestModule.getDefault(); + public AbstractFileStringExtract(KeywordSearchIngestModule module) { + this.module = module; this.ingester = Server.getIngester(); this.extractScripts.add(DEFAULT_SCRIPT); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java index 04b4ad00b9..4f52805753 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java @@ -35,7 +35,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ReadContentInputStream; import org.apache.tika.Tika; @@ -57,7 +56,7 @@ import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; */ class AbstractFileTikaTextExtract implements AbstractFileExtract { - private static final Logger logger = Logger.getLogger(IngestModuleAbstractFile.class.getName()); + private static final Logger logger = Logger.getLogger(AbstractFileTikaTextExtract.class.getName()); private static final Charset OUTPUT_CHARSET = Server.DEFAULT_INDEXED_TEXT_CHARSET; static final int MAX_EXTR_TEXT_CHARS = 512 * 1024; private static final int SINGLE_READ_CHARS = 1024; @@ -72,8 +71,8 @@ class AbstractFileTikaTextExtract implements AbstractFileExtract { private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor(); private final List TIKA_SUPPORTED_TYPES = new ArrayList<>(); - AbstractFileTikaTextExtract() { - this.module = KeywordSearchIngestModule.getDefault(); + AbstractFileTikaTextExtract(KeywordSearchIngestModule module) { + this.module = module; ingester = Server.getIngester(); Set mediaTypes = new Tika().getParser().getSupportedTypes(new ParseContext()); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java index 3bdd226d17..54779a4dab 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java @@ -86,10 +86,10 @@ abstract class AbstractKeywordSearchPerformer extends javax.swing.JPanel impleme @Override public void search() { - boolean isRunning = IngestManager.getDefault().isModuleRunning(KeywordSearchIngestModule.getDefault()); + boolean isIngestRunning = IngestManager.getDefault().isIngestRunning(); if (filesIndexed == 0) { - if (isRunning) { + if (isIngestRunning) { KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.noFilesInIdxMsg", KeywordSearchSettings.getUpdateFrequency().getTime()), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); @@ -101,7 +101,7 @@ abstract class AbstractKeywordSearchPerformer extends javax.swing.JPanel impleme } //check if keyword search module ingest is running (indexing, etc) - if (isRunning) { + if (isIngestRunning) { if (KeywordSearchUtil.displayConfirmDialog(org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle"), NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.ingestInProgressBody"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN) == false) { return; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Keyword.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Keyword.java index 65acb171b1..5c14fbafdd 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Keyword.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Keyword.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -76,8 +76,6 @@ class Keyword { return NbBundle.getMessage(this.getClass(), "Keyword.toString.text", keywordString, isLiteral, keywordType); } - - @Override public boolean equals(Object obj) { if (obj == null) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java new file mode 100755 index 0000000000..093d9462f1 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java @@ -0,0 +1,118 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.Date; +import java.util.List; + +public class KeywordList { + + private String name; + private Date created; + private Date modified; + private Boolean useForIngest; + private Boolean ingestMessages; + private List keywords; + private Boolean locked; + + KeywordList(String name, Date created, Date modified, Boolean useForIngest, Boolean ingestMessages, List keywords, boolean locked) { + this.name = name; + this.created = created; + this.modified = modified; + this.useForIngest = useForIngest; + this.ingestMessages = ingestMessages; + this.keywords = keywords; + this.locked = locked; + } + + KeywordList(String name, Date created, Date modified, Boolean useForIngest, Boolean ingestMessages, List keywords) { + this(name, created, modified, useForIngest, ingestMessages, keywords, false); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final KeywordList other = (KeywordList) obj; + if ((this.name == null) ? (other.name != null) : !this.name.equals(other.name)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int hash = 5; + return hash; + } + + String getName() { + return name; + } + + Date getDateCreated() { + return created; + } + + Date getDateModified() { + return modified; + } + + Boolean getUseForIngest() { + return useForIngest; + } + + void setUseForIngest(boolean use) { + this.useForIngest = use; + } + + Boolean getIngestMessages() { + return ingestMessages; + } + + void setIngestMessages(boolean ingestMessages) { + this.ingestMessages = ingestMessages; + } + + List getKeywords() { + return keywords; + } + + boolean hasKeyword(Keyword keyword) { + return keywords.contains(keyword); + } + + boolean hasKeyword(String keyword) { + //note, this ignores isLiteral + for (Keyword k : keywords) { + if (k.getQuery().equals(keyword)) { + return true; + } + } + return false; + } + + Boolean isLocked() { + return locked; + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordListsManager.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordListsManager.java new file mode 100755 index 0000000000..ddddf76115 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordListsManager.java @@ -0,0 +1,101 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; + +/** + * Keeps track, by name, of the keyword lists to be used for file ingest. + */ +// Note: This is a first step towards a keyword lists manager; it consists of +// the portion of the keyword list management code that resided in the keyword +// search file ingest module. +final class KeywordListsManager { + + private static KeywordListsManager instance = null; + private final Logger logger = Logger.getLogger(KeywordListsManager.class.getName()); + private final List keywordListNames = new ArrayList<>(); + private final List keywords = new ArrayList<>(); + + /** + * Gets the keyword lists manager singleton. + */ + static synchronized KeywordListsManager getInstance() { + if (null == instance) { + instance = new KeywordListsManager(); + } + return instance; + } + + private KeywordListsManager() { + addKeywordListsForFileIngest(null); + } + + /** + * Sets the keyword lists to be used for ingest. The lists that are used + * will be the union of the lists enabled using the keyword search global + * options panel and a selection, possibly empty, of the disabled lists. + * + * @param listNames The names of disabled lists to temporarily enable + */ + synchronized void addKeywordListsForFileIngest(List listNames) { + keywords.clear(); + keywordListNames.clear(); + + StringBuilder logMessage = new StringBuilder(); + KeywordSearchListsXML globalKeywordSearchOptions = KeywordSearchListsXML.getCurrent(); + for (KeywordList list : globalKeywordSearchOptions.getListsL()) { + String listName = list.getName(); + if ((list.getUseForIngest() == true) || (listNames != null && listNames.contains(listName))) { + keywordListNames.add(listName); + logMessage.append(listName).append(" "); + } + + for (Keyword keyword : list.getKeywords()) { + if (!keywords.contains(keyword)) { + keywords.add(keyword); + } + } + } + + logger.log(Level.INFO, "Keyword lists for file ingest set to: {0}", logMessage.toString()); + } + + /** + * Returns the keyword lists to be used for ingest, by name. + * + * @return The names of the enabled keyword lists + */ + synchronized List getNamesOfKeywordListsForFileIngest() { + return new ArrayList<>(keywordListNames); + } + + /** + * Indicates whether or not there are currently keywords for which to search + * during ingest. + * + * @return True if there are no keywords specified, false otherwise + */ + synchronized boolean hasNoKeywordsForSearch() { + return (keywords.isEmpty()); + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel.java index 0b33a5aea8..a738a98ebb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 - 2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,16 +16,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.keywordsearch; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; +import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSetttingsPanel; /** - * Container panel for keyword search advanced configuration options + * Global options panel for keyword searching. */ -public final class KeywordSearchConfigurationPanel extends javax.swing.JPanel implements OptionsPanel { +final class KeywordSearchConfigurationPanel extends IngestModuleGlobalSetttingsPanel implements OptionsPanel { private KeywordSearchConfigurationPanel1 listsPanel; private KeywordSearchConfigurationPanel3 languagesPanel; @@ -35,7 +35,7 @@ public final class KeywordSearchConfigurationPanel extends javax.swing.JPanel im initComponents(); customizeComponents(); } - + private void customizeComponents() { setName(NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel.customizeComponents.title")); listsPanel = new KeywordSearchConfigurationPanel1(); @@ -71,38 +71,37 @@ public final class KeywordSearchConfigurationPanel extends javax.swing.JPanel im ); }// //GEN-END:initComponents - /** - * Load each of the tabs and reload the XML. - */ @Override public void load() { - // Deselect all table rows + // This calls actually clears the component. listsPanel.load(); + languagesPanel.load(); generalPanel.load(); + // Reload the XML to avoid 'ghost' vars KeywordSearchListsXML.getCurrent().reload(); } - /** - * Store each panel's settings. - */ @Override - public void store() { + public void saveSettings() { listsPanel.store(); languagesPanel.store(); - generalPanel.store(); + generalPanel.store(); + } + + @Override + public void store() { + saveSettings(); } public void cancel() { - KeywordSearchListsXML.getCurrent().reload(); + KeywordSearchListsXML.getCurrent().reload(); } - + boolean valid() { - // TODO check whether form is consistent and complete return true; } - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JTabbedPane tabbedPane; // End of variables declaration//GEN-END:variables diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel1.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel1.java index 34bb43a679..2ad85d59bb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel1.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel1.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,18 +17,12 @@ * limitations under the License. */ -/* - * KeywordSearchConfigurationPanel1.java - * - * Created on Feb 28, 2012, 4:12:47 PM - */ package org.sleuthkit.autopsy.keywordsearch; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.util.List; import javax.swing.JOptionPane; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; @@ -43,9 +37,7 @@ class KeywordSearchConfigurationPanel1 extends javax.swing.JPanel implements Opt private static final Logger logger = Logger.getLogger(KeywordSearchConfigurationPanel1.class.getName()); private static final String KEYWORD_CONFIG_NAME = org.openide.util.NbBundle.getMessage(KeywordSearchPanel.class, "ListBundleConfig"); - /** Creates new form KeywordSearchConfigurationPanel1 */ - KeywordSearchConfigurationPanel1() { - + KeywordSearchConfigurationPanel1() { initComponents(); customizeComponents(); setName(KEYWORD_CONFIG_NAME); @@ -80,7 +72,7 @@ class KeywordSearchConfigurationPanel1 extends javax.swing.JPanel implements Opt public void actionPerformed(ActionEvent e) { final String FEATURE_NAME = "Save Keyword List"; KeywordSearchListsXML writer = KeywordSearchListsXML.getCurrent(); - KeywordSearchListsAbstract.KeywordSearchList currentKeywordList = editListPanel.getCurrentKeywordList(); + KeywordList currentKeywordList = editListPanel.getCurrentKeywordList(); List keywords = currentKeywordList.getKeywords(); if (keywords.isEmpty()) { @@ -121,8 +113,6 @@ class KeywordSearchConfigurationPanel1 extends javax.swing.JPanel implements Opt writer.addList(listName, keywords); KeywordSearchUtil.displayDialog(FEATURE_NAME, NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.kwListSavedMsg", listName), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.INFO); } - - //currentKeywordList = writer.getList(listName); listsManagementPanel.resync(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel2.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel2.java index 641b61caac..1d5497839b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel2.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel2.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012 Basis Technology Corp. + * Copyright 2012-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,6 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.apache.solr.client.solrj.SolrServerException; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.UpdateFrequency; @@ -45,8 +44,7 @@ class KeywordSearchConfigurationPanel2 extends javax.swing.JPanel implements Opt private void activateWidgets() { skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown()); showSnippetsCB.setSelected(KeywordSearchSettings.getShowSnippets()); - boolean enable = !IngestManager.getDefault().isIngestRunning() - && !IngestManager.getDefault().isModuleRunning(KeywordSearchIngestModule.getDefault()); + boolean enable = !IngestManager.getDefault().isIngestRunning(); skipNSRLCheckBox.setEnabled(enable); setTimeSettingEnabled(enable); @@ -70,7 +68,6 @@ class KeywordSearchConfigurationPanel2 extends javax.swing.JPanel implements Opt timeRadioButton3.setSelected(true); break; } - } /** @@ -205,7 +202,6 @@ class KeywordSearchConfigurationPanel2 extends javax.swing.JPanel implements Opt .addContainerGap(116, Short.MAX_VALUE)) ); }// //GEN-END:initComponents - // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JLabel chunksLabel; private javax.swing.JLabel chunksValLabel; @@ -248,14 +244,11 @@ class KeywordSearchConfigurationPanel2 extends javax.swing.JPanel implements Opt private UpdateFrequency getSelectedTimeValue() { if (timeRadioButton1.isSelected()) { return UpdateFrequency.FAST; - } - else if (timeRadioButton2.isSelected()) { + } else if (timeRadioButton2.isSelected()) { return UpdateFrequency.AVG; - } - else if (timeRadioButton3.isSelected()) { + } else if (timeRadioButton3.isSelected()) { return UpdateFrequency.SLOW; - } - else if (timeRadioButton4.isSelected()) { + } else if (timeRadioButton4.isSelected()) { return UpdateFrequency.SLOWEST; } return UpdateFrequency.DEFAULT; @@ -273,36 +266,28 @@ class KeywordSearchConfigurationPanel2 extends javax.swing.JPanel implements Opt try { filesIndexedValue.setText(Integer.toString(KeywordSearch.getServer().queryNumIndexedFiles())); chunksValLabel.setText(Integer.toString(KeywordSearch.getServer().queryNumIndexedChunks())); - } catch (KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Could not get number of indexed files/chunks"); - - } catch (NoOpenCoreException ex) { + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { logger.log(Level.WARNING, "Could not get number of indexed files/chunks"); } KeywordSearch.addNumIndexedFilesChangeListener( new PropertyChangeListener() { - @Override - public void propertyChange(PropertyChangeEvent evt) { - String changed = evt.getPropertyName(); - Object newValue = evt.getNewValue(); + @Override + public void propertyChange(PropertyChangeEvent evt) { + String changed = evt.getPropertyName(); + Object newValue = evt.getNewValue(); - if (changed.equals(KeywordSearch.NUM_FILES_CHANGE_EVT)) { - int newFilesIndexed = ((Integer) newValue).intValue(); - filesIndexedValue.setText(Integer.toString(newFilesIndexed)); - try { - chunksValLabel.setText(Integer.toString(KeywordSearch.getServer().queryNumIndexedChunks())); - } catch (KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Could not get number of indexed chunks"); + if (changed.equals(KeywordSearch.NUM_FILES_CHANGE_EVT)) { + int newFilesIndexed = ((Integer) newValue).intValue(); + filesIndexedValue.setText(Integer.toString(newFilesIndexed)); + try { + chunksValLabel.setText(Integer.toString(KeywordSearch.getServer().queryNumIndexedChunks())); + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + logger.log(Level.WARNING, "Could not get number of indexed chunks"); - } catch (NoOpenCoreException ex) { - logger.log(Level.WARNING, "Could not get number of indexed chunks"); - } - - } } - }); - - + } + } + }); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel3.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel3.java index ff736bfc2e..cd74942a82 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel3.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchConfigurationPanel3.java @@ -149,8 +149,7 @@ class KeywordSearchConfigurationPanel3 extends javax.swing.JPanel implements Opt enableUTF8Checkbox.setSelected(utf8); final boolean extractEnabled = utf16 || utf8; - boolean ingestNotRunning = !IngestManager.getDefault().isIngestRunning() - && ! IngestManager.getDefault().isModuleRunning(KeywordSearchIngestModule.getDefault()); + boolean ingestNotRunning = !IngestManager.getDefault().isIngestRunning() && !IngestManager.getDefault().isIngestRunning(); //enable / disable checboxes activateScriptsCheckboxes(extractEnabled && ingestNotRunning); enableUTF16Checkbox.setEnabled(ingestNotRunning); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java index 2cf89f27c5..623fc73cfa 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,6 @@ * limitations under the License. */ - package org.sleuthkit.autopsy.keywordsearch; import java.awt.Component; @@ -30,7 +29,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.logging.Level; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import java.util.regex.Pattern; @@ -49,7 +47,7 @@ import javax.swing.table.TableCellRenderer; import javax.swing.table.TableColumn; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent; +import org.sleuthkit.autopsy.ingest.IngestManager.IngestEvent; import org.sleuthkit.datamodel.BlackboardAttribute; /** @@ -59,7 +57,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec private static Logger logger = Logger.getLogger(KeywordSearchEditListPanel.class.getName()); private KeywordTableModel tableModel; - private KeywordSearchListsAbstract.KeywordSearchList currentKeywordList; + private KeywordList currentKeywordList; private boolean ingestRunning; @@ -161,7 +159,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec - if (IngestManager.getDefault().isModuleRunning(KeywordSearchIngestModule.getDefault())) { + if (IngestManager.getDefault().isIngestRunning()) { initIngest(0); } else { initIngest(1); @@ -173,14 +171,14 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); Object oldValue = evt.getOldValue(); - if (changed.equals(IngestModuleEvent.COMPLETED.toString() ) - && ((String) oldValue).equals(KeywordSearchIngestModule.MODULE_NAME)) { + if (changed.equals(IngestEvent.COMPLETED.toString() ) + && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { initIngest(1); - } else if (changed.equals(IngestModuleEvent.STARTED.toString() ) - && ((String) oldValue).equals(KeywordSearchIngestModule.MODULE_NAME)) { + } else if (changed.equals(IngestEvent.STARTED.toString() ) + && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { initIngest(0); - } else if (changed.equals(IngestModuleEvent.STOPPED.toString() ) - && ((String) oldValue).equals(KeywordSearchIngestModule.MODULE_NAME)) { + } else if (changed.equals(IngestEvent.STOPPED.toString() ) + && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { initIngest(1); } } @@ -221,9 +219,9 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec boolean noKeywords = !listSet ? true : currentKeywordList.getKeywords().isEmpty(); // Certain buttons will be disabled if ingest is ongoing on this list - List ingestLists = new ArrayList(); + List ingestLists = new ArrayList<>(); if (ingestOngoing) { - ingestLists = KeywordSearchIngestModule.getDefault().getKeywordLists(); + ingestLists = KeywordListsManager.getInstance().getNamesOfKeywordListsForFileIngest(); } boolean inIngest = !listSet ? false : ingestLists.contains(currentKeywordList.getName()); @@ -575,7 +573,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec KeywordSearchListsXML reader = KeywordSearchListsXML.getCurrent(); - List toWrite = new ArrayList(); + List toWrite = new ArrayList(); toWrite.add(reader.getList(currentKeywordList.getName())); final KeywordSearchListsXML exporter = new KeywordSearchListsXML(fileAbs); boolean written = exporter.saveLists(toWrite); @@ -659,11 +657,11 @@ private void useForIngestCheckboxActionPerformed(java.awt.event.ActionEvent evt) // Implemented by parent panel } - KeywordSearchListsAbstract.KeywordSearchList getCurrentKeywordList() { + KeywordList getCurrentKeywordList() { return currentKeywordList; } - void setCurrentKeywordList(KeywordSearchListsAbstract.KeywordSearchList list) { + void setCurrentKeywordList(KeywordList list) { currentKeywordList = list; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index f84fe746ba..78da5b0f1f 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -34,7 +34,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; - import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.SwingUtilities; @@ -50,13 +49,12 @@ import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.StopWatch; import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT; -import org.sleuthkit.autopsy.coreutils.Version; -import org.sleuthkit.autopsy.ingest.PipelineContext; +import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; -import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile; -import org.sleuthkit.autopsy.ingest.IngestModuleInit; +import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -76,10 +74,8 @@ import org.sleuthkit.datamodel.TskData.FileKnown; * ingest update interval) Runs a periodic keyword / regular expression search * on currently configured lists for ingest and writes results to blackboard * Reports interesting events to Inbox and to viewers - * - * Registered as a module in layer.xml */ -public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { +public final class KeywordSearchIngestModule extends IngestModuleAdapter implements FileIngestModule { enum UpdateFrequency { @@ -99,19 +95,10 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { } }; private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName()); - public static final String MODULE_NAME = NbBundle.getMessage(KeywordSearchIngestModule.class, - "KeywordSearchIngestModule.moduleName"); - public static final String MODULE_DESCRIPTION = NbBundle.getMessage(KeywordSearchIngestModule.class, - "KeywordSearchIngestModule.moduleDescription"); - final public static String MODULE_VERSION = Version.getVersion(); - private static KeywordSearchIngestModule instance = null; - private IngestServices services; + private IngestServices services = IngestServices.getDefault(); private Ingester ingester = null; private volatile boolean commitIndex = false; //whether to commit index next time private volatile boolean runSearcher = false; //whether to run searcher next time - private List keywords; //keywords to search - private List keywordLists; // lists currently being searched - private Map keywordToList; //keyword to list name mapping private Timer commitTimer; private Timer searchTimer; private Indexer indexer; @@ -124,52 +111,126 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { private Set curDataSourceIds; private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy private static final Lock searcherLock = rwLock.writeLock(); - private volatile int messageID = 0; + private volatile int messageID = 0; // RJCTODO: Despite volatile, this is not thread safe, uses increment (not atomic) private boolean processedFiles; private volatile boolean finalSearcherDone = true; //mark as done, until it's inited - private final String hashDBModuleName = NbBundle - .getMessage(this.getClass(), "KeywordSearchIngestModule.hashDbModuleName"); //NOTE this needs to match the HashDB module getName() private SleuthkitCase caseHandle = null; private static List textExtractors; private static AbstractFileStringExtract stringExtractor; private boolean initialized = false; - private KeywordSearchIngestSimplePanel simpleConfigPanel; - private KeywordSearchConfigurationPanel advancedConfigPanel; private Tika tikaFormatDetector; - private enum IngestStatus { - TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested + + TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested STRINGS_INGESTED, ///< Strings were extracted from file - METADATA_INGESTED, ///< No content, so we just text_ingested metadata + METADATA_INGESTED, ///< No content, so we just text_ingested metadata SKIPPED_ERROR_INDEXING, ///< File was skipped because index engine had problems SKIPPED_ERROR_TEXTEXTRACT, ///< File was skipped because of text extraction issues SKIPPED_ERROR_IO ///< File was skipped because of IO issues reading it }; private Map ingestStatus; - //private constructor to ensure singleton instance - private KeywordSearchIngestModule() { + KeywordSearchIngestModule() { } /** - * Returns singleton instance of the module, creates one if needed + * Initializes the module for new ingest run Sets up threads, timers, + * retrieves settings, keyword lists to run on * - * @return instance of the module */ - public static synchronized KeywordSearchIngestModule getDefault() { - if (instance == null) { - instance = new KeywordSearchIngestModule(); + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + logger.log(Level.INFO, "init()"); + initialized = false; + + caseHandle = Case.getCurrentCase().getSleuthkitCase(); + + tikaFormatDetector = new Tika(); + + ingester = Server.getIngester(); + + final Server server = KeywordSearch.getServer(); + try { + if (!server.isRunning()) { + String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); + logger.log(Level.SEVERE, msg); + String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); + services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details)); + throw new IngestModuleException(msg); + } + } catch (KeywordSearchModuleException ex) { + logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex); + //this means Solr is not properly initialized + String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); + String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); + services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details)); + throw new IngestModuleException(msg); } - return instance; + try { + // make an actual query to verify that server is responding + // we had cases where getStatus was OK, but the connection resulted in a 404 + server.queryNumIndexedDocuments(); + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + throw new IngestModuleException( + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg", + ex.getMessage())); + } + + //initialize extractors + stringExtractor = new AbstractFileStringExtract(this); + stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); + stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); + + //log the scripts used for debugging + final StringBuilder sbScripts = new StringBuilder(); + for (SCRIPT s : KeywordSearchSettings.getStringExtractScripts()) { + sbScripts.append(s.name()).append(" "); + } + logger.log(Level.INFO, "Using string extract scripts: {0}", sbScripts.toString()); + + textExtractors = new ArrayList<>(); + //order matters, more specific extractors first + textExtractors.add(new AbstractFileHtmlExtract(this)); + textExtractors.add(new AbstractFileTikaTextExtract(this)); + + ingestStatus = new HashMap<>(); + + if (KeywordListsManager.getInstance().hasNoKeywordsForSearch()) { + services.postMessage(IngestMessage.createWarningMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); + + } + + processedFiles = false; + finalSearcherDone = false; + searcherDone = true; //make sure to start the initial currentSearcher + //keeps track of all results per run not to repeat reporting the same hits + currentResults = new HashMap<>(); + + curDataSourceIds = new HashSet<>(); + + indexer = new Indexer(); + + final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; + logger.log(Level.INFO, "Using commit interval (ms): {0}", updateIntervalMs); + logger.log(Level.INFO, "Using searcher interval (ms): {0}", updateIntervalMs); + + commitTimer = new Timer(updateIntervalMs, new CommitTimerAction()); + searchTimer = new Timer(updateIntervalMs, new SearchTimerAction()); + + initialized = true; + + commitTimer.start(); + searchTimer.start(); } @Override - public ProcessResult process(PipelineContext pipelineContext, AbstractFile abstractFile) { + public ProcessResult process(AbstractFile abstractFile) { if (initialized == false) //error initializing indexing/Solr { - logger.log(Level.WARNING, "Skipping processing, module not initialized, file: " + abstractFile.getName()); + logger.log(Level.WARNING, "Skipping processing, module not initialized, file: {0}", abstractFile.getName()); ingestStatus.put(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); return ProcessResult.OK; } @@ -181,26 +242,19 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex); } - + if (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.VIRTUAL_DIR)) { //skip indexing of virtual dirs (no content, no real name) - will index children files return ProcessResult.OK; } - //check if we should index meta-data only when 1) it is known 2) HashDb module errored on it - if (services.getAbstractFileModuleResult(hashDBModuleName) == IngestModuleAbstractFile.ProcessResult.ERROR) { - indexer.indexFile(abstractFile, false); - //notify depending module that keyword search (would) encountered error for this file - ingestStatus.put(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_IO); - return ProcessResult.ERROR; - } - else if (KeywordSearchSettings.getSkipKnown() && abstractFile.getKnown().equals(FileKnown.KNOWN)) { + if (KeywordSearchSettings.getSkipKnown() && abstractFile.getKnown().equals(FileKnown.KNOWN)) { //index meta-data only indexer.indexFile(abstractFile, false); return ProcessResult.OK; } - processedFiles = true; + processedFiles = true; //check if it's time to commit after previous processing checkRunCommitSearch(); @@ -216,12 +270,16 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { * Cleanup resources, threads, timers */ @Override - public void complete() { + public void shutDown(boolean ingestJobCancelled) { if (initialized == false) { return; } - //logger.log(Level.INFO, "complete()"); + if (ingestJobCancelled) { + stop(); + return; + } + commitTimer.stop(); //NOTE, we let the 1 before last searcher complete fully, and enqueue the last one @@ -240,7 +298,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { postIndexSummary(); //run one last search as there are probably some new files committed - if (keywordLists != null && !keywordLists.isEmpty() && processedFiles == true) { + List keywordLists = KeywordListsManager.getInstance().getNamesOfKeywordListsForFileIngest(); + if (!keywordLists.isEmpty() && processedFiles == true) { finalSearcher = new Searcher(keywordLists, true); //final searcher run finalSearcher.execute(); } else { @@ -252,24 +311,19 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { try { final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); final int numIndexedChunks = KeywordSearch.getServer().queryNumIndexedChunks(); - logger.log(Level.INFO, "Indexed files count: " + numIndexedFiles); - logger.log(Level.INFO, "Indexed file chunks count: " + numIndexedChunks); - } catch (NoOpenCoreException ex) { + logger.log(Level.INFO, "Indexed files count: {0}", numIndexedFiles); + logger.log(Level.INFO, "Indexed file chunks count: {0}", numIndexedChunks); + } catch (NoOpenCoreException | KeywordSearchModuleException ex) { logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex); - } catch (KeywordSearchModuleException se) { - logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", se); } //cleanup done in final searcher - - //postSummary(); } /** * Handle stop event (ingest interrupted) Cleanup resources, threads, timers */ - @Override - public void stop() { + private void stop() { logger.log(Level.INFO, "stop()"); //stop timer @@ -286,12 +340,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { runSearcher = false; finalSearcherDone = true; - //commit uncommited files, don't search again commit(); - //postSummary(); - cleanup(); } @@ -314,197 +365,11 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { textExtractors = null; stringExtractor = null; - keywords.clear(); - keywordLists.clear(); - keywordToList.clear(); - tikaFormatDetector = null; initialized = false; } - @Override - public String getName() { - return MODULE_NAME; - } - - @Override - public String getDescription() { - return MODULE_DESCRIPTION; - } - - @Override - public String getVersion() { - return MODULE_VERSION; - } - - /** - * Initializes the module for new ingest run Sets up threads, timers, - * retrieves settings, keyword lists to run on - * - */ - @Override - public void init(IngestModuleInit initContext) throws IngestModuleException { - logger.log(Level.INFO, "init()"); - services = IngestServices.getDefault(); - initialized = false; - - caseHandle = Case.getCurrentCase().getSleuthkitCase(); - - tikaFormatDetector = new Tika(); - - ingester = Server.getIngester(); - - final Server server = KeywordSearch.getServer(); - try { - if (!server.isRunning()) { - String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); - logger.log(Level.SEVERE, msg); - String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); - services.postMessage(IngestMessage.createErrorMessage(++messageID, instance, msg, details)); - throw new IngestModuleException(msg); - } - } catch (KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex); - //this means Solr is not properly initialized - String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); - String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); - services.postMessage(IngestMessage.createErrorMessage(++messageID, instance, msg, details)); - throw new IngestModuleException(msg); - } - try { - // make an actual query to verify that server is responding - // we had cases where getStatus was OK, but the connection resulted in a 404 - server.queryNumIndexedDocuments(); - } catch (KeywordSearchModuleException | NoOpenCoreException ex) { - throw new IngestModuleException( - NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg", - ex.getMessage())); - } - - //initialize extractors - stringExtractor = new AbstractFileStringExtract(); - stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); - stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); - - - //log the scripts used for debugging - final StringBuilder sbScripts = new StringBuilder(); - for (SCRIPT s : KeywordSearchSettings.getStringExtractScripts()) { - sbScripts.append(s.name()).append(" "); - } - logger.log(Level.INFO, "Using string extract scripts: " + sbScripts.toString()); - - textExtractors = new ArrayList(); - //order matters, more specific extractors first - textExtractors.add(new AbstractFileHtmlExtract()); - textExtractors.add(new AbstractFileTikaTextExtract()); - - - ingestStatus = new HashMap(); - - keywords = new ArrayList(); - keywordLists = new ArrayList(); - keywordToList = new HashMap(); - - initKeywords(); - - if (keywords.isEmpty() || keywordLists.isEmpty()) { - services.postMessage(IngestMessage.createWarningMessage(++messageID, instance, NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"), - NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg"))); - } - - processedFiles = false; - finalSearcherDone = false; - searcherDone = true; //make sure to start the initial currentSearcher - //keeps track of all results per run not to repeat reporting the same hits - currentResults = new HashMap>(); - - curDataSourceIds = new HashSet(); - - indexer = new Indexer(); - - final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000; - logger.log(Level.INFO, "Using commit interval (ms): " + updateIntervalMs); - logger.log(Level.INFO, "Using searcher interval (ms): " + updateIntervalMs); - - commitTimer = new Timer(updateIntervalMs, new CommitTimerAction()); - searchTimer = new Timer(updateIntervalMs, new SearchTimerAction()); - - initialized = true; - - commitTimer.start(); - searchTimer.start(); - } - - @Override - public boolean hasSimpleConfiguration() { - return true; - } - - @Override - public boolean hasAdvancedConfiguration() { - return true; - } - - @Override - public javax.swing.JPanel getSimpleConfiguration(String context) { - KeywordSearchListsXML.getCurrent().reload(); - - if (null == simpleConfigPanel) { - simpleConfigPanel = new KeywordSearchIngestSimplePanel(); - } - else { - simpleConfigPanel.load(); - } - - return simpleConfigPanel; - } - - @Override - public javax.swing.JPanel getAdvancedConfiguration(String context) { - if (advancedConfigPanel == null) { - advancedConfigPanel = new KeywordSearchConfigurationPanel(); - } - - advancedConfigPanel.load(); - return advancedConfigPanel; - } - - @Override - public void saveAdvancedConfiguration() { - if (advancedConfigPanel != null) { - advancedConfigPanel.store(); - } - - if (simpleConfigPanel != null) { - simpleConfigPanel.load(); - } - } - - @Override - public void saveSimpleConfiguration() { - KeywordSearchListsXML.getCurrent().save(); - } - - /** - * The modules maintains background threads, return true if background - * threads are running or there are pending tasks to be run in the future, - * such as the final search post-ingest completion - * - * @return - */ - @Override - public boolean hasBackgroundJobsRunning() { - if ((currentSearcher != null && searcherDone == false) - || (finalSearcherDone == false)) { - return true; - } else { - return false; - } - - } - /** * Commits index and notifies listeners of index update */ @@ -562,13 +427,12 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { msg.append("").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.errIoLbl")).append("").append(error_io).append(""); msg.append(""); String indexStats = msg.toString(); - logger.log(Level.INFO, "Keyword Indexing Completed: " + indexStats); - services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, this, NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats)); + logger.log(Level.INFO, "Keyword Indexing Completed: {0}", indexStats); + services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats)); if (error_index > 0) { MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrsTitle"), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrMsgFiles", error_index)); - } - else if (error_io + error_text > 0) { + } else if (error_io + error_text > 0) { MessageNotifyUtil.Notify.warn(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxWarnMsgTitle"), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.idxErrReadFilesMsg")); } @@ -582,64 +446,11 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { try { final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles)); - } catch (NoOpenCoreException ex) { + } catch (NoOpenCoreException | KeywordSearchModuleException ex) { logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex); - } catch (KeywordSearchModuleException se) { - logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", se); } } - /** - * Initialize the keyword search lists and associated keywords from the XML - * loader Use the lists to ingest that are set in the permanent XML - * configuration - */ - private void initKeywords() { - addKeywordLists(null); - } - - /** - * If ingest is ongoing, this will add additional keyword search lists to - * the ongoing ingest The lists to add may be temporary and not necessary - * set to be added to ingest permanently in the XML configuration. The lists - * will be reset back to original (permanent configuration state) on the - * next ingest. - * - * @param listsToAdd lists to add temporarily to the ongoing ingest - */ - void addKeywordLists(List listsToAdd) { - KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent(); - - keywords.clear(); - keywordLists.clear(); - keywordToList.clear(); - - StringBuilder sb = new StringBuilder(); - - for (KeywordSearchListsAbstract.KeywordSearchList list : loader.getListsL()) { - final String listName = list.getName(); - if (list.getUseForIngest() == true - || (listsToAdd != null && listsToAdd.contains(listName))) { - keywordLists.add(listName); - sb.append(listName).append(" "); - } - for (Keyword keyword : list.getKeywords()) { - if (!keywords.contains(keyword)) { - keywords.add(keyword); - keywordToList.put(keyword.getQuery(), list); - } - } - - } - - logger.log(Level.INFO, "Set new effective keyword lists: " + sb.toString()); - - } - - List getKeywordLists() { - return keywordLists == null ? new ArrayList() : keywordLists; - } - /** * Check if time to commit, if so, run commit. Then run search if search * timer is also set. @@ -655,7 +466,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { //in worst case, we will run search next time after commit timer goes off, or at the end of ingest if (searcherDone && runSearcher) { //start search if previous not running - if (keywordLists != null && !keywordLists.isEmpty()) { + List keywordLists = KeywordListsManager.getInstance().getNamesOfKeywordListsForFileIngest(); + if (!keywordLists.isEmpty()) { currentSearcher = new Searcher(keywordLists); currentSearcher.execute();//searcher will stop timer and restart timer when done } @@ -724,8 +536,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { } if (fileExtract == null) { - logger.log(Level.INFO, "No text extractor found for file id:" - + aFile.getId() + ", name: " + aFile.getName() + ", detected format: " + detectedFormat); + logger.log(Level.INFO, "No text extractor found for file id:{0}, name: {1}, detected format: {2}", new Object[]{aFile.getId(), aFile.getName(), detectedFormat}); return false; } @@ -748,7 +559,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { ingestStatus.put(aFile.getId(), IngestStatus.STRINGS_INGESTED); return true; } else { - logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ")."); + logger.log(Level.WARNING, "Failed to extract strings and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); return false; } @@ -782,14 +593,14 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { * Adds the file to the index. Detects file type, calls extractors, etc. * * @param aFile File to analyze - * @param indexContent False if only metadata should be text_ingested. True if - * content and metadata should be index. + * @param indexContent False if only metadata should be text_ingested. + * True if content and metadata should be index. */ private void indexFile(AbstractFile aFile, boolean indexContent) { //logger.log(Level.INFO, "Processing AbstractFile: " + abstractFile.getName()); - TskData.TSK_DB_FILES_TYPE_ENUM aType = aFile.getType(); - + TskData.TSK_DB_FILES_TYPE_ENUM aType = aFile.getType(); + // unallocated and unused blocks can only have strings extracted from them. if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) || aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))) { extractStringsAndIndex(aFile); @@ -801,8 +612,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { try { ingester.ingest(aFile, false); //meta-data only ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); - } - catch (IngesterException ex) { + } catch (IngesterException ex) { ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); } @@ -815,11 +625,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { try { is = new ReadContentInputStream(aFile); detectedFormat = tikaFormatDetector.detect(is, aFile.getName()); - } - catch (Exception e) { + } catch (Exception e) { logger.log(Level.WARNING, "Could not detect format using tika for file: " + aFile, e); - } - finally { + } finally { if (is != null) { try { is.close(); @@ -829,9 +637,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { } } } - + // @@@ Add file type signature to blackboard here - + //logger.log(Level.INFO, "Detected format: " + aFile.getName() + " " + detectedFormat); // we skip archive formats that are opened by the archive module. @@ -840,8 +648,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { try { ingester.ingest(aFile, false); //meta-data only ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); - } - catch (IngesterException ex) { + } catch (IngesterException ex) { ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); } @@ -854,7 +661,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { try { //logger.log(Level.INFO, "indexing: " + aFile.getName()); if (!extractTextAndIndex(aFile, detectedFormat)) { - logger.log(Level.WARNING, "Failed to extract text and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ")."); + logger.log(Level.WARNING, "Failed to extract text and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } else { ingestStatus.put(aFile.getId(), IngestStatus.TEXT_INGESTED); @@ -892,15 +699,15 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { */ private List keywords; //keywords to search private List keywordLists; // lists currently being searched - private Map keywordToList; //keyword to list name mapping + private Map keywordToList; //keyword to list name mapping private AggregateProgressHandle progressGroup; private final Logger logger = Logger.getLogger(Searcher.class.getName()); private boolean finalRun = false; Searcher(List keywordLists) { - this.keywordLists = new ArrayList(keywordLists); - this.keywords = new ArrayList(); - this.keywordToList = new HashMap(); + this.keywordLists = new ArrayList<>(keywordLists); + this.keywords = new ArrayList<>(); + this.keywordToList = new HashMap<>(); //keywords are populated as searcher runs } @@ -917,15 +724,15 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { logger.log(Level.INFO, "Pending start of new searcher"); } - final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + - (finalRun ? (" - "+ NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); - progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" ("+ - NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") +")"), null, new Cancellable() { + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" (" + + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"), null, new Cancellable() { @Override public boolean cancel() { logger.log(Level.INFO, "Cancelling the searcher by user."); if (progressGroup != null) { - progressGroup.setDisplayName(displayName + " ("+ NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") +"...)"); + progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") + "...)"); } return Searcher.this.cancel(true); } @@ -965,12 +772,12 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { for (Keyword keywordQuery : keywords) { if (this.isCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: " + keywordQuery.getQuery()); + logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery()); return null; } final String queryStr = keywordQuery.getQuery(); - final KeywordSearchListsAbstract.KeywordSearchList list = keywordToList.get(queryStr); + final KeywordList list = keywordToList.get(queryStr); final String listName = list.getName(); //new subProgress will be active after the initial query @@ -985,10 +792,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { boolean isRegex = !keywordQuery.isLiteral(); if (isRegex) { del = new TermComponentQuery(keywordQuery); - } - else { + } else { del = new LuceneQuery(keywordQuery); - del.escape(); + del.escape(); } //limit search to currently ingested data sources @@ -996,7 +802,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds); del.addFilter(dataSourceFilter); - Map> queryResult = null; + Map> queryResult; try { queryResult = del.performQuery(); @@ -1007,7 +813,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { //likely case has closed and threads are being interrupted return null; } catch (CancellationException e) { - logger.log(Level.INFO, "Cancel detected, bailing during keyword query: " + keywordQuery.getQuery()); + logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery()); return null; } catch (Exception e) { logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); @@ -1023,7 +829,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { //write results to BB //new artifacts created, to report to listeners - Collection newArtifacts = new ArrayList(); + Collection newArtifacts = new ArrayList<>(); //scale progress bar more more granular, per result sub-progress, within per keyword int totalUnits = newResults.size(); @@ -1040,10 +846,10 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { for (final Keyword hitTerm : newResults.keySet()) { //checking for cancellation between results if (this.isCancelled()) { - logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: " + keywordQuery.getQuery()); + logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: {0}", keywordQuery.getQuery()); return null; } - + // update progress display String hitDisplayStr = hitTerm.getQuery(); if (hitDisplayStr.length() > 50) { @@ -1055,9 +861,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { // this returns the unique files in the set with the first chunk that has a hit Map contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm)); for (final AbstractFile hitFile : contentHitsFlattened.keySet()) { - + // get the snippet for the first hit in the file - String snippet = null; + String snippet; final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery()); int chunkId = contentHitsFlattened.get(hitFile); try { @@ -1074,7 +880,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { // write the blackboard artifact for this keyword in this file KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName); if (written == null) { - logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: " + hitFile + ", hit: " + hitTerm.toString()); + logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hitFile, hitTerm.toString()}); continue; } @@ -1148,8 +954,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { } } detailsSb.append(""); - - services.postMessage(IngestMessage.createDataMessage(++messageID, instance, subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); + + services.postMessage(IngestMessage.createDataMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); } } //for each file hit @@ -1159,7 +965,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { //update artifact browser if (!newArtifacts.isEmpty()) { - services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); + services.fireModuleDataEvent(new ModuleDataEvent(KeywordSearchModuleFactory.getModuleName(), ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); } } //if has results @@ -1177,7 +983,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { try { finalizeSearcher(); stopWatch.stop(); - logger.log(Level.INFO, "Searcher took to run: " + stopWatch.getElapsedTimeSecs() + " secs."); + logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs()); } finally { searcherLock.unlock(); } @@ -1185,16 +991,15 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { return null; } - + @Override protected void done() { // call get to see if there were any errors try { get(); - } - catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException | ExecutionException e) { logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage()); - services.postMessage(IngestMessage.createErrorMessage(++messageID, instance, "Error performing keyword search", e.getMessage())); + services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage())); } // catch and ignore if we were cancelled catch (java.util.concurrent.CancellationException ex ) { } @@ -1210,7 +1015,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { this.keywordToList.clear(); for (String name : this.keywordLists) { - KeywordSearchListsAbstract.KeywordSearchList list = loader.getList(name); + KeywordList list = loader.getList(name); for (Keyword k : list.getKeywords()) { this.keywords.add(k); this.keywordToList.put(k.getQuery(), list); @@ -1261,13 +1066,13 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { //calculate new results but substracting results already obtained in this ingest //update currentResults map with the new results private Map> filterResults(Map> queryResult, boolean isRegex) { - Map> newResults = new HashMap>(); + Map> newResults = new HashMap<>(); for (String termResult : queryResult.keySet()) { List queryTermResults = queryResult.get(termResult); //translate to list of IDs that we keep track of - List queryTermResultsIDs = new ArrayList(); + List queryTermResultsIDs = new ArrayList<>(); for (ContentHit ch : queryTermResults) { queryTermResultsIDs.add(ch.getId()); } @@ -1284,7 +1089,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAbstractFile { //add to new results List newResultsFs = newResults.get(termResultK); if (newResultsFs == null) { - newResultsFs = new ArrayList(); + newResultsFs = new ArrayList<>(); newResults.put(termResultK, newResultsFs); } newResultsFs.add(res); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java index 7c6b8090d6..7576e901ef 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011 - 2013 Basis Technology Corp. + * Copyright 2011 - 2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,6 @@ * limitations under the License. */ - package org.sleuthkit.autopsy.keywordsearch; import java.util.ArrayList; @@ -28,21 +27,23 @@ import javax.swing.ListSelectionModel; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableColumn; import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT; +import org.sleuthkit.autopsy.ingest.IngestModuleSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleSettingsPanel; +import org.sleuthkit.autopsy.ingest.NoIngestModuleSettings; /** - * Simple ingest config panel + * Ingest job options panel for the keyword search file ingest module. */ -public class KeywordSearchIngestSimplePanel extends javax.swing.JPanel { +public class KeywordSearchIngestSimplePanel extends IngestModuleSettingsPanel { private final static Logger logger = Logger.getLogger(KeywordSearchIngestSimplePanel.class.getName()); public static final String PROP_OPTIONS = "Keyword Search_Options"; private KeywordTableModel tableModel; - private List lists; + private List lists; - /** Creates new form KeywordSearchIngestSimplePanel */ - public KeywordSearchIngestSimplePanel() { + KeywordSearchIngestSimplePanel() { tableModel = new KeywordTableModel(); - lists = new ArrayList(); + lists = new ArrayList<>(); reloadLists(); initComponents(); customizeComponents(); @@ -56,7 +57,7 @@ public class KeywordSearchIngestSimplePanel extends javax.swing.JPanel { //customize column witdhs final int width = listsScrollPane.getPreferredSize().width; listsTable.setAutoResizeMode(JTable.AUTO_RESIZE_NEXT_COLUMN); - TableColumn column = null; + TableColumn column; for (int i = 0; i < listsTable.getColumnCount(); i++) { column = listsTable.getColumnModel().getColumn(i); if (i == 0) { @@ -70,6 +71,11 @@ public class KeywordSearchIngestSimplePanel extends javax.swing.JPanel { reloadEncodings(); } + @Override + public IngestModuleSettings getSettings() { + return new NoIngestModuleSettings(); + } + public void load() { KeywordSearchListsXML.getCurrent().reload(); reloadLists(); @@ -183,7 +189,6 @@ public class KeywordSearchIngestSimplePanel extends javax.swing.JPanel { // End of variables declaration//GEN-END:variables private void reloadLangs() { - //TODO multiple List