diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index cc95e5b986..837612fa53 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -16,6 +16,9 @@ IngestMessagePanel.totalMessagesNameLabel.text=Total: IngestMessagePanel.totalMessagesNameVal.text=- IngestMessagePanel.totalUniqueMessagesNameLabel.text=Unique: IngestMessagePanel.totalUniqueMessagesNameVal.text=- +IngestJob.progress.dataSourceIngest.displayName=Data Source Ingest of {0} +IngestJob.progress.fileIngest.displayName=File Ingest of {0} +IngestJob.progress.cancelling={0} (Cancelling...) IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=Processes unallocated space, such as deleted files. Produces more complete results, but it may take longer to process on large images. IngestJobConfigurationPanel.processUnallocCheckbox.text=Process Unallocated Space IngestJobConfigurationPanel.advancedButton.text=Advanced @@ -46,14 +49,8 @@ IngestManager.toHtmlStr.totalErrs.text=Total errors\: {0} IngestManager.toHtmlStr.module.text=Module IngestManager.toHtmlStr.time.text=Time IngestManager.toHtmlStr.errors.text=Errors -IngestManager.FileTaskWorker.displayName=File Ingest -IngestManager.FileTaskWorker.process.cancelling={0} (Cancelling...) IngestManager.EnqueueWorker.displayName.text=Queueing Ingest IngestManager.EnqueueWorker.process.cancelling={0} (Cancelling...) -IngestManager.DataSourceTaskWorker.progress.pending={0} (Pending...) -IngestManager.DataSourceTaskWorker.progress.cancelling={0} (Cancelling...) -IngestManager.datatSourceIngest.progress.text=DataSource Ingest {0} -IngestManager.fileIngest.progress.text=File Ingest {0} IngestJob.DataSourceIngestPipeline.displayName.text={0} processing {1} IngestMessage.toString.type.text=type\: {0} IngestMessage.toString.source.text=\ source\: {0} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java index 1a6350164a..689d8e7652 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java @@ -31,11 +31,13 @@ public class DataSourceIngestModuleStatusHelper { private final SwingWorker worker; private final ProgressHandle progress; private final Content dataSource; + private final String moduleDisplayName; - DataSourceIngestModuleStatusHelper(SwingWorker worker, ProgressHandle progress, Content dataSource) { + DataSourceIngestModuleStatusHelper(SwingWorker worker, ProgressHandle progress, Content dataSource, String moduleDisplayName) { this.worker = worker; this.progress = progress; this.dataSource = dataSource; + this.moduleDisplayName = moduleDisplayName; } /** @@ -57,7 +59,7 @@ public class DataSourceIngestModuleStatusHelper { * @param workUnits Total number of work units for the processing of the * data source. */ - public void switchToDeterminate(int workUnits) { // RJCTODO: Fix this + public void switchToDeterminate(int workUnits) { if (progress != null) { progress.switchToDeterminate(workUnits); } @@ -81,7 +83,7 @@ public class DataSourceIngestModuleStatusHelper { */ public void progress(int workUnits) { if (progress != null) { - progress.progress(dataSource.getName(), workUnits); + progress.progress(this.moduleDisplayName, workUnits); } } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java new file mode 100755 index 0000000000..3b919391bc --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -0,0 +1,148 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import javax.swing.SwingWorker; +import org.netbeans.api.progress.ProgressHandle; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Content; + +/** + * A data source ingest pipeline composed of a sequence of data source ingest + * modules constructed from ingest module templates. + */ +final class DataSourceIngestPipeline { + + private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName()); + private final IngestJob job; + private final List moduleTemplates; + private List modules = new ArrayList<>(); + + DataSourceIngestPipeline(IngestJob task, List moduleTemplates) { + this.job = task; + this.moduleTemplates = moduleTemplates; + } + + List startUp() { + List errors = new ArrayList<>(); + // Create an ingest module instance from each ingest module template + // that has an ingest module factory capable of making data source + // ingest modules. Map the module class names to the module instance + // to allow the modules to be put in the sequence indicated by the + // ingest pipelines configuration. + Map modulesByClass = new HashMap<>(); + for (IngestModuleTemplate template : moduleTemplates) { + if (template.isDataSourceIngestModuleTemplate()) { + DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); + IngestJobContext context = new IngestJobContext(job); + try { + module.startUp(context); + modulesByClass.put(module.getClassName(), module); + IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), module.getDisplayName()); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + } + } + // Establish the module sequence of the core ingest modules + // indicated by the ingest pipeline configuration, adding any + // additional modules found in the global lookup to the end of the + // pipeline in arbitrary order. + List pipelineConfig = IngestPipelinesConfiguration.getInstance().getDataSourceIngestPipelineConfig(); + for (String moduleClassName : pipelineConfig) { + if (modulesByClass.containsKey(moduleClassName)) { + modules.add(modulesByClass.remove(moduleClassName)); + } + } + for (DataSourceIngestModuleDecorator module : modulesByClass.values()) { + modules.add(module); + } + return errors; + } + + List process(SwingWorker worker, ProgressHandle progress) { + List errors = new ArrayList<>(); + Content dataSource = this.job.getDataSource(); + logger.log(Level.INFO, "Processing data source {0}", dataSource.getName()); + for (DataSourceIngestModuleDecorator module : this.modules) { + try { + module.process(dataSource, new DataSourceIngestModuleStatusHelper(worker, progress, dataSource, module.getDisplayName())); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (job.isCancelled()) { + break; + } + } + return errors; + } + + List shutDown(boolean ingestJobCancelled) { + List errors = new ArrayList<>(); + for (DataSourceIngestModuleDecorator module : this.modules) { + try { + module.shutDown(ingestJobCancelled); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } finally { + IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); + } + } + return errors; + } + + private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { + + private final DataSourceIngestModule module; + private final String displayName; + + DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { + this.module = module; + this.displayName = displayName; + } + + String getClassName() { + return module.getClass().getCanonicalName(); + } + + String getDisplayName() { + return displayName; + } + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + module.startUp(context); + } + + @Override + public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) { + return module.process(dataSource, statusHelper); + } + + @Override + public void shutDown(boolean ingestJobWasCancelled) { + module.shutDown(ingestJobWasCancelled); + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java new file mode 100755 index 0000000000..dac583bd4f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -0,0 +1,149 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; + +/** + * A file ingest pipeline composed of a sequence of file ingest modules + * constructed from ingest module templates. + */ +final class FileIngestPipeline { + + private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName()); + private final IngestJob job; + private final List moduleTemplates; + private List modules = new ArrayList<>(); + + FileIngestPipeline(IngestJob task, List moduleTemplates) { + this.job = task; + this.moduleTemplates = moduleTemplates; + } + + List startUp() { + List errors = new ArrayList<>(); + // Create an ingest module instance from each ingest module template + // that has an ingest module factory capable of making data source + // ingest modules. Map the module class names to the module instance + // to allow the modules to be put in the sequence indicated by the + // ingest pipelines configuration. + Map modulesByClass = new HashMap<>(); + for (IngestModuleTemplate template : moduleTemplates) { + if (template.isFileIngestModuleTemplate()) { + FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); + IngestJobContext context = new IngestJobContext(job); + try { + module.startUp(context); + modulesByClass.put(module.getClassName(), module); + IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), template.getModuleName()); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + } + } + // Establish the module sequence of the core ingest modules + // indicated by the ingest pipeline configuration, adding any + // additional modules found in the global lookup to the end of the + // pipeline in arbitrary order. + List pipelineConfig = IngestPipelinesConfiguration.getInstance().getFileIngestPipelineConfig(); + for (String moduleClassName : pipelineConfig) { + if (modulesByClass.containsKey(moduleClassName)) { + modules.add(modulesByClass.remove(moduleClassName)); + } + } + for (FileIngestModuleDecorator module : modulesByClass.values()) { + modules.add(module); + } + return errors; + } + + List process(AbstractFile file) { + List errors = new ArrayList<>(); + Content dataSource = this.job.getDataSource(); + logger.log(Level.INFO, String.format("Processing {0} from {1}", file.getName(), dataSource.getName())); + for (FileIngestModuleDecorator module : this.modules) { + try { + module.process(file); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (job.isCancelled()) { + break; + } + } + file.close(); + IngestManager.fireFileDone(file.getId()); + return errors; + } + + List shutDown(boolean ingestJobCancelled) { + List errors = new ArrayList<>(); + for (FileIngestModuleDecorator module : this.modules) { + try { + module.shutDown(ingestJobCancelled); + } catch (Exception ex) { + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } finally { + IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); + } + } + return errors; + } + + private static class FileIngestModuleDecorator implements FileIngestModule { + + private final FileIngestModule module; + private final String displayName; + + FileIngestModuleDecorator(FileIngestModule module, String displayName) { + this.module = module; + this.displayName = displayName; + } + + String getClassName() { + return module.getClass().getCanonicalName(); + } + + String getDisplayName() { + return displayName; + } + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + module.startUp(context); + } + + @Override + public IngestModule.ProcessResult process(AbstractFile file) { + return module.process(file); + } + + @Override + public void shutDown(boolean ingestJobWasCancelled) { + module.shutDown(ingestJobWasCancelled); + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index f2d7ade02e..25f28fe3c3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -21,13 +21,10 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; +import org.netbeans.api.progress.ProgressHandleFactory; +import org.openide.util.Cancellable; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** @@ -42,8 +39,13 @@ final class IngestJob { private final boolean processUnallocatedSpace; private final HashMap fileIngestPipelines = new HashMap<>(); private final HashMap dataSourceIngestPipelines = new HashMap<>(); + private final IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); private FileIngestPipeline initialFileIngestPipeline = null; private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; + private ProgressHandle dataSourceTaskProgress; + private ProgressHandle fileTasksProgress; + int totalEnqueuedFiles = 0; + private int processedFiles = 0; private boolean cancelled; IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { @@ -66,15 +68,53 @@ final class IngestJob { return processUnallocatedSpace; } - synchronized void cancel() { - cancelled = true; - } - - synchronized boolean isCancelled() { - return cancelled; - } - synchronized List startUpIngestPipelines() { + startDataSourceIngestProgressBar(); + startFileIngestProgressBar(); + return startUpInitialIngestPipelines(); + } + + private void startDataSourceIngestProgressBar() { + final String displayName = NbBundle + .getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", this.dataSource.getName()); + dataSourceTaskProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (dataSourceTaskProgress != null) { + dataSourceTaskProgress.setDisplayName(NbBundle.getMessage(this.getClass(), + "IngestJob.progress.cancelling", + displayName)); + } + IngestManager.getInstance().stopAll(); + return true; + } + }); + dataSourceTaskProgress.start(); + dataSourceTaskProgress.switchToIndeterminate(); + } + + private void startFileIngestProgressBar() { + final String displayName = NbBundle + .getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); + fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (fileTasksProgress != null) { + fileTasksProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", + displayName)); + } + IngestManager.getInstance().stopAll(); + return true; + } + }); + fileTasksProgress.start(); + fileTasksProgress.switchToIndeterminate(); + totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); + fileTasksProgress.switchToDeterminate(totalEnqueuedFiles); + } + + private List startUpInitialIngestPipelines() { // Create a per thread instance of each pipeline type right now to make // (reasonably) sure that the ingest modules can be started. initialDataSourceIngestPipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); @@ -124,13 +164,19 @@ final class IngestJob { if (dataSourceIngestPipeline != null) { errors.addAll(dataSourceIngestPipeline.shutDown(cancelled)); } - this.dataSourceIngestPipelines.remove(threadId); + dataSourceIngestPipelines.remove(threadId); + if (dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) { + dataSourceTaskProgress.finish(); + } FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId); if (fileIngestPipeline != null) { errors.addAll(fileIngestPipeline.shutDown(cancelled)); } - this.fileIngestPipelines.remove(threadId); + fileIngestPipelines.remove(threadId); + if (fileIngestPipelines.isEmpty() && fileTasksProgress != null) { + fileTasksProgress.finish(); + } return errors; } @@ -139,251 +185,29 @@ final class IngestJob { return (dataSourceIngestPipelines.isEmpty() && fileIngestPipelines.isEmpty()); } - /** - * A data source ingest pipeline composed of a sequence of data source ingest - * modules constructed from ingest module templates. - */ - static final class DataSourceIngestPipeline { - - private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName()); - private final IngestJob task; - private final List moduleTemplates; - private List modules = new ArrayList<>(); - - private DataSourceIngestPipeline(IngestJob task, List moduleTemplates) { - this.task = task; - this.moduleTemplates = moduleTemplates; - } - - private List startUp() { - List errors = new ArrayList<>(); - - // Create an ingest module instance from each ingest module template - // that has an ingest module factory capable of making data source - // ingest modules. Map the module class names to the module instance - // to allow the modules to be put in the sequence indicated by the - // ingest pipelines configuration. - Map modulesByClass = new HashMap<>(); - for (IngestModuleTemplate template : moduleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); - IngestJobContext context = new IngestJobContext(task); - try { - module.startUp(context); - modulesByClass.put(module.getClassName(), module); - IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), module.getDisplayName()); - } catch (Exception ex) { - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - } - } - - // Establish the module sequence of the core ingest modules - // indicated by the ingest pipeline configuration, adding any - // additional modules found in the global lookup to the end of the - // pipeline in arbitrary order. - List pipelineConfig = IngestPipelinesConfiguration.getInstance().getDataSourceIngestPipelineConfig(); - for (String moduleClassName : pipelineConfig) { - if (modulesByClass.containsKey(moduleClassName)) { - modules.add(modulesByClass.remove(moduleClassName)); - } - } - for (DataSourceIngestModuleDecorator module : modulesByClass.values()) { - modules.add(module); - } - - return errors; - } - - List process(SwingWorker worker, ProgressHandle progress) { - List errors = new ArrayList<>(); - Content dataSource = this.task.getDataSource(); - logger.log(Level.INFO, "Processing data source {0}", dataSource.getName()); - for (DataSourceIngestModuleDecorator module : this.modules) { - try { - String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.DataSourceIngestPipeline.displayName.text", module.getDisplayName(), dataSource.getName()); - progress.setDisplayName(displayName); - module.process(dataSource, new DataSourceIngestModuleStatusHelper(worker, progress, dataSource)); - } catch (Exception ex) { - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - if (task.isCancelled()) { - break; - } - } - return errors; - } - - private List shutDown(boolean ingestJobCancelled) { - List errors = new ArrayList<>(); - for (DataSourceIngestModuleDecorator module : this.modules) { - try { - module.shutDown(ingestJobCancelled); - } catch (Exception ex) { - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } finally { - IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); - } - } - return errors; - } - - private static class DataSourceIngestModuleDecorator implements DataSourceIngestModule { - - private final DataSourceIngestModule module; - private final String displayName; - - DataSourceIngestModuleDecorator(DataSourceIngestModule module, String displayName) { - this.module = module; - this.displayName = displayName; - } - - String getClassName() { - return module.getClass().getCanonicalName(); - } - - String getDisplayName() { - return displayName; - } - - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); - } - - @Override - public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleStatusHelper statusHelper) { - return module.process(dataSource, statusHelper); - } - - @Override - public void shutDown(boolean ingestJobWasCancelled) { - module.shutDown(ingestJobWasCancelled); - } - } + synchronized ProgressHandle getDataSourceTaskProgressBar() { + return this.dataSourceTaskProgress; } - /** - * A file ingest pipeline composed of a sequence of file ingest modules - * constructed from ingest module templates. - */ - static final class FileIngestPipeline { - - private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName()); - private final IngestJob task; - private final List moduleTemplates; - private List modules = new ArrayList<>(); - - private FileIngestPipeline(IngestJob task, List moduleTemplates) { - this.task = task; - this.moduleTemplates = moduleTemplates; + synchronized void handleFileTaskStarted(IngestScheduler.FileScheduler.FileTask task) { + int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); + if (newTotalEnqueuedFiles > totalEnqueuedFiles) { + totalEnqueuedFiles = newTotalEnqueuedFiles + 1; + fileTasksProgress.switchToIndeterminate(); + fileTasksProgress.switchToDeterminate(totalEnqueuedFiles); + } + if (processedFiles < totalEnqueuedFiles) { + ++processedFiles; } - private List startUp() { - List errors = new ArrayList<>(); - - // Create an ingest module instance from each ingest module template - // that has an ingest module factory capable of making data source - // ingest modules. Map the module class names to the module instance - // to allow the modules to be put in the sequence indicated by the - // ingest pipelines configuration. - Map modulesByClass = new HashMap<>(); - for (IngestModuleTemplate template : moduleTemplates) { - if (template.isFileIngestModuleTemplate()) { - FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); // RJCTODO: Move into try block for bpth impls - IngestJobContext context = new IngestJobContext(task); - try { - module.startUp(context); - modulesByClass.put(module.getClassName(), module); - IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), template.getModuleName()); - } catch (Exception ex) { - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - } - } - - // Establish the module sequence of the core ingest modules - // indicated by the ingest pipeline configuration, adding any - // additional modules found in the global lookup to the end of the - // pipeline in arbitrary order. - List pipelineConfig = IngestPipelinesConfiguration.getInstance().getFileIngestPipelineConfig(); - for (String moduleClassName : pipelineConfig) { - if (modulesByClass.containsKey(moduleClassName)) { - modules.add(modulesByClass.remove(moduleClassName)); - } - } - for (FileIngestModuleDecorator module : modulesByClass.values()) { - modules.add(module); - } - - return errors; - } + fileTasksProgress.progress(task.getFile().getName(), processedFiles); + } - List process(AbstractFile file) { - List errors = new ArrayList<>(); - Content dataSource = this.task.getDataSource(); - logger.log(Level.INFO, String.format("Processing {0} from {1}", file.getName(), dataSource.getName())); - for (FileIngestModuleDecorator module : this.modules) { - try { - module.process(file); - } catch (Exception ex) { - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - if (task.isCancelled()) { - break; - } - } - file.close(); - IngestManager.fireFileDone(file.getId()); - return errors; - } + synchronized void cancel() { + cancelled = true; + } - private List shutDown(boolean ingestJobCancelled) { - List errors = new ArrayList<>(); - for (FileIngestModuleDecorator module : this.modules) { - try { - module.shutDown(ingestJobCancelled); - } catch (Exception ex) { - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } finally { - IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); - } - } - return errors; - } - - private static class FileIngestModuleDecorator implements FileIngestModule { - - private final FileIngestModule module; - private final String displayName; - - FileIngestModuleDecorator(FileIngestModule module, String displayName) { - this.module = module; - this.displayName = displayName; - } - - String getClassName() { - return module.getClass().getCanonicalName(); - } - - String getDisplayName() { - return displayName; - } - - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); - } - - @Override - public IngestModule.ProcessResult process(AbstractFile file) { - return module.process(file); - } - - @Override - public void shutDown(boolean ingestJobWasCancelled) { - module.shutDown(ingestJobWasCancelled); - } - } - } + synchronized boolean isCancelled() { + return cancelled; + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 5e3746fe5b..4cd4958b0e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -488,7 +488,6 @@ public class IngestManager { class DataSourceTaskWorker extends SwingWorker { private final long id; - private ProgressHandle progress; DataSourceTaskWorker(long threadId) { this.id = threadId; @@ -497,25 +496,6 @@ public class IngestManager { @Override protected Void doInBackground() throws Exception { logger.log(Level.INFO, "Data source ingest thread (id={0}) started", this.id); - - // Set up a progress bar that can be used to cancel all of the - // ingest jobs currently being performed. - progress = ProgressHandleFactory.createHandle("Data source ingest", new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", DataSourceTaskWorker.this.id); - if (progress != null) { - progress.setDisplayName(NbBundle.getMessage(this.getClass(), - "IngestManager.DataSourceTaskWorker.process.cancelling", - "Data source ingest")); - } - IngestManager.getInstance().stopAll(); - return true; - } - }); - progress.start(); - progress.switchToIndeterminate(); - IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler(); while (scheduler.hasNext()) { if (isCancelled()) { @@ -523,13 +503,11 @@ public class IngestManager { return null; } - IngestJob ingestJob = scheduler.next(); - IngestJob.DataSourceIngestPipeline pipeline = ingestJob.getDataSourceIngestPipelineForThread(this.id); - pipeline.process(this, this.progress); + IngestJob job = scheduler.next(); + DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(this.id); + pipeline.process(this, job.getDataSourceTaskProgressBar()); } - logger.log(Level.INFO, "Data source ingest thread (id={0}) completed", this.id); - IngestManager.getInstance().reportThreadDone(this.id); return null; } @@ -539,13 +517,11 @@ public class IngestManager { super.get(); } catch (CancellationException | InterruptedException e) { logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id); - IngestManager.getInstance().reportThreadDone(this.id); } catch (Exception ex) { String message = String.format("Data source ingest thread (id=%d) experienced a fatal error", this.id); logger.log(Level.SEVERE, message, ex); - IngestManager.getInstance().reportThreadDone(this.id); } finally { - progress.finish(); + IngestManager.getInstance().reportThreadDone(this.id); } } } @@ -557,7 +533,6 @@ public class IngestManager { class FileTaskWorker extends SwingWorker { private final long id; - private ProgressHandle progress; FileTaskWorker(long threadId) { this.id = threadId; @@ -566,58 +541,20 @@ public class IngestManager { @Override protected Object doInBackground() throws Exception { logger.log(Level.INFO, "File ingest thread (id={0}) started", this.id); - - // Set up a progress bar that can be used to cancel all of the - // ingest jobs currently being performed. - final String displayName = NbBundle - .getMessage(this.getClass(), "IngestManager.FileTaskWorker.displayName"); - progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", FileTaskWorker.this.id); - if (progress != null) { - progress.setDisplayName( - NbBundle.getMessage(this.getClass(), "IngestManager.FileTaskWorker.process.cancelling", - displayName)); - } - IngestManager.getInstance().stopAll(); - return true; - } - }); - progress.start(); - progress.switchToIndeterminate(); IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); - int totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); - progress.switchToDeterminate(totalEnqueuedFiles); - - int processedFiles = 0; while (fileScheduler.hasNext()) { if (isCancelled()) { IngestManager.getInstance().reportThreadDone(this.id); logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id); return null; } - IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); - AbstractFile file = task.getFile(); - progress.progress(file.getName(), processedFiles); - IngestJob.FileIngestPipeline pipeline = task.getJob().getFileIngestPipelineForThread(this.id); - pipeline.process(file); - - // Update the progress bar. - int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); - if (newTotalEnqueuedFiles > totalEnqueuedFiles) { - totalEnqueuedFiles = newTotalEnqueuedFiles + 1; - progress.switchToIndeterminate(); - progress.switchToDeterminate(totalEnqueuedFiles); - } - if (processedFiles < totalEnqueuedFiles) { - ++processedFiles; - } + IngestJob job = task.getJob(); + FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(this.id); + job.handleFileTaskStarted(task); + pipeline.process(task.getFile()); } - logger.log(Level.INFO, "File ingest thread (id={0}) completed", this.id); - IngestManager.getInstance().reportThreadDone(this.id); return null; } @@ -627,13 +564,11 @@ public class IngestManager { super.get(); } catch (CancellationException | InterruptedException e) { logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id); - IngestManager.getInstance().reportThreadDone(this.id); } catch (Exception ex) { String message = String.format("File ingest thread {0} experienced a fatal error", this.id); logger.log(Level.SEVERE, message, ex); - IngestManager.getInstance().reportThreadDone(this.id); } finally { - progress.finish(); + IngestManager.getInstance().reportThreadDone(this.id); } } }