diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java index d1328ef68c..cbf38d7453 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java @@ -162,8 +162,7 @@ final class ArtifactsListPanel extends AbstractArtifactListPanel { @ThreadConfined(type = ThreadConfined.ThreadType.AWT) @Override void clearList() { - tableModel.setContents(new ArrayList<>()); - tableModel.fireTableDataChanged(); + addArtifacts(new ArrayList<>()); } /** diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java index c50ab50020..496141b091 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java @@ -80,9 +80,9 @@ final class DomainDetailsPanel extends JPanel { if (selectedTabName == null || !selectedTabName.equals(newTabTitle)) { selectedTabName = newTabTitle; Component selectedComponent = jTabbedPane1.getSelectedComponent(); - if (selectedComponent instanceof DomainArtifactsTabPanel) { + if (!StringUtils.isBlank(domain) && selectedComponent instanceof DomainArtifactsTabPanel) { runDomainWorker((DomainArtifactsTabPanel) selectedComponent, true); - } else if (selectedComponent instanceof MiniTimelinePanel) { + } else if (!StringUtils.isBlank(domain) && selectedComponent instanceof MiniTimelinePanel) { runMiniTimelineWorker((MiniTimelinePanel) selectedComponent, true); } } @@ -170,13 +170,13 @@ final class DomainDetailsPanel extends JPanel { @Subscribe void handlePopulateDomainTabsEvent(DiscoveryEventUtils.PopulateDomainTabsEvent populateEvent) { SwingUtilities.invokeLater(() -> { - if (StringUtils.isBlank(populateEvent.getDomain())) { + domain = populateEvent.getDomain(); + if (StringUtils.isBlank(domain)) { resetTabsStatus(); //send fade out event DiscoveryEventUtils.getDiscoveryEventBus().post(new DiscoveryEventUtils.DetailsVisibleEvent(false)); } else { resetTabsStatus(); - domain = populateEvent.getDomain(); Component selectedComponent = jTabbedPane1.getSelectedComponent(); if (selectedComponent instanceof DomainArtifactsTabPanel) { runDomainWorker((DomainArtifactsTabPanel) selectedComponent, false); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java index 6f79817782..83b5a1457f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java @@ -23,6 +23,11 @@ import org.sleuthkit.datamodel.Content; /** * An adapter that provides a no-op implementation of the startUp() method for * data source ingest modules. + * + * NOTE: As of Java 8, interfaces can have default methods. + * DataSourceIngestModule now provides default no-op versions of startUp() and + * shutDown(). This class is no longer needed and can be deprecated when + * convenient. */ public abstract class DataSourceIngestModuleAdapter implements DataSourceIngestModule { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index c22d63348c..6c551da602 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2011-2016 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,184 +18,86 @@ */ package org.sleuthkit.autopsy.ingest; -import java.util.ArrayList; -import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source level ingest modules for an - * ingestJobPipeline. It starts the modules, runs data sources through them, and - * shuts them down when data source level ingest is complete. - *

- * This class is thread-safe. + * A pipeline of data source level ingest modules for performing data source + * level ingest tasks for an ingest job. */ -final class DataSourceIngestPipeline { +final class DataSourceIngestPipeline extends IngestTaskPipeline { - private static final IngestManager ingestManager = IngestManager.getInstance(); private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName()); - private final IngestJobPipeline ingestJobPipeline; - private final List modules = new ArrayList<>(); - private volatile PipelineModule currentModule; + private static final IngestManager ingestManager = IngestManager.getInstance(); /** - * Constructs an object that manages a sequence of data source level ingest - * modules. It starts the modules, runs data sources through them, and shuts - * them down when data source level ingest is complete. + * Constructs a pipeline of data source level ingest modules for performing + * data source level ingest tasks for an ingest job. * - * @param ingestJobPipeline The ingestJobPipeline that owns this pipeline. - * @param moduleTemplates Templates for the creating the ingest modules that - * make up this pipeline. + * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. */ DataSourceIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { - this.ingestJobPipeline = ingestJobPipeline; - for (IngestModuleTemplate template : moduleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); - modules.add(module); - } - } + super(ingestJobPipeline, moduleTemplates); } - /** - * Indicates whether or not there are any ingest modules in this pipeline. - * - * @return True or false. - */ - boolean isEmpty() { - return modules.isEmpty(); + @Override + Optional> acceptModuleTemplate(IngestModuleTemplate template) { + Optional> module = Optional.empty(); + if (template.isDataSourceIngestModuleTemplate()) { + DataSourceIngestModule ingestModule = template.createDataSourceIngestModule(); + module = Optional.of(new DataSourcePipelineModule(ingestModule, template.getModuleName())); + } + return module; } - /** - * Starts up the ingest modules in this pipeline. - * - * @return A list of ingest module startup errors, possibly empty. - */ - synchronized List startUp() { - List errors = new ArrayList<>(); - for (PipelineModule module : modules) { - try { - module.startUp(new IngestJobContext(this.ingestJobPipeline)); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - } - return errors; + @Override + void prepareTask(DataSourceIngestTask task) { } - /** - * Runs a data source through the ingest modules in sequential order. - * - * @param task A data source level ingest task containing a data source to - * be processed. - * - * @return A list of processing errors, possible empty. - */ - synchronized List process(DataSourceIngestTask task) { - List errors = new ArrayList<>(); - if (!this.ingestJobPipeline.isCancelled()) { - Content dataSource = task.getDataSource(); - for (PipelineModule module : modules) { - try { - this.currentModule = module; - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.displayName", - module.getDisplayName(), dataSource.getName()); - this.ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(displayName); - this.ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate(); - DataSourceIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); - logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) starting", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS - module.process(dataSource, new DataSourceIngestModuleProgress(this.ingestJobPipeline)); - logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) finished", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - if (this.ingestJobPipeline.isCancelled()) { - break; - } else if (this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) { - this.ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(currentModule.getDisplayName()); - } - } - } - this.currentModule = null; + @Override + void completeTask(DataSourceIngestTask task) { ingestManager.setIngestTaskProgressCompleted(task); - return errors; } /** - * Gets the currently running module. - * - * @return The module, possibly null if no module is currently running. + * A wrapper that adds ingest infrastructure operations to a data source + * level ingest module. */ - PipelineModule getCurrentlyRunningModule() { - return this.currentModule; - } - - /** - * This class decorates a data source level ingest module with a display - * name and a processing start time. - */ - static class PipelineModule implements DataSourceIngestModule { + static final class DataSourcePipelineModule extends IngestTaskPipeline.PipelineModule { private final DataSourceIngestModule module; - private final String displayName; - private volatile Date processingStartTime; /** - * Constructs an object that decorates a data source level ingest module - * with a display name and a processing start time. - * - * @param module The data source level ingest module to be - * decorated. - * @param displayName The display name. + * Constructs a wrapper that adds ingest infrastructure operations to a + * data source level ingest module. */ - PipelineModule(DataSourceIngestModule module, String displayName) { + DataSourcePipelineModule(DataSourceIngestModule module, String displayName) { + super(module, displayName); this.module = module; - this.displayName = displayName; - this.processingStartTime = new Date(); - } - - /** - * Gets the class name of the decorated ingest module. - * - * @return The class name. - */ - String getClassName() { - return this.module.getClass().getCanonicalName(); - } - - /** - * Gets the display of the decorated ingest module. - * - * @return The display name. - */ - String getDisplayName() { - return this.displayName; - } - - /** - * Gets the time the decorated ingest module started processing the data - * source. - * - * @return The start time, will be null if the module has not started - * processing the data source yet. - */ - Date getProcessingStartTime() { - return this.processingStartTime; } @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - this.module.startUp(context); - } - - @Override - public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - this.processingStartTime = new Date(); - return this.module.process(dataSource, statusHelper); + void performTask(IngestJobPipeline ingestJobPipeline, DataSourceIngestTask task) throws IngestModuleException { + Content dataSource = task.getDataSource(); + String progressBarDisplayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", getDisplayName(), dataSource.getName()); + ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(progressBarDisplayName); + ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate(); + ingestManager.setIngestTaskProgress(task, getDisplayName()); + logger.log(Level.INFO, "{0} analysis of {1} starting", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS + ProcessResult result = module.process(dataSource, new DataSourceIngestModuleProgress(ingestJobPipeline)); + logger.log(Level.INFO, "{0} analysis of {1} finished", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS + if (!ingestJobPipeline.isCancelled() && ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) { + ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(getDisplayName()); + } + if (result == ProcessResult.ERROR) { + throw new IngestModuleException(String.format("%s experienced an error analyzing %s (data source objId = %d)", getDisplayName(), dataSource.getName(), dataSource.getId())); //NON-NLS + } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java index 215c9e0bcb..60355e10f8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,13 +36,4 @@ public interface FileIngestModule extends IngestModule { */ ProcessResult process(AbstractFile file); - /** - * Invoked by Autopsy when an ingest job is completed (either because the - * data has been analyzed or because the job was canceled - check - * IngestJobContext.fileIngestIsCancelled()), before the ingest module - * instance is discarded. The module should respond by doing things like - * releasing private resources, submitting final results, and posting a - * final ingest message. - */ - void shutDown(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java index 13811e6eb4..9f733308c8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java @@ -23,6 +23,10 @@ import org.sleuthkit.datamodel.AbstractFile; /** * An adapter that provides no-op implementations of the startUp() and * shutDown() methods for file ingest modules. + * + * NOTE: As of Java 8, interfaces can have default methods. FileIngestModule now + * provides default no-op versions of startUp() and shutDown(). This class is no + * longer needed and can be deprecated when convenient. */ public abstract class FileIngestModuleAdapter implements FileIngestModule { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 2f6604e415..1cc6ce05a4 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2014-2015 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,223 +18,109 @@ */ package org.sleuthkit.autopsy.ingest; -import java.util.ArrayList; -import java.util.Date; import java.util.List; -import java.util.logging.Level; - -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import java.util.Optional; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * This class manages a sequence of file level ingest modules for an - * ingest job pipeline. It starts the modules, runs files through them, and shuts them - * down when file level ingest is complete. - *

- * This class is thread-safe. + * A pipeline of file ingest modules for performing file ingest tasks for an + * ingest job. */ -final class FileIngestPipeline { +final class FileIngestPipeline extends IngestTaskPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobPipeline ingestJobPipeline; - private final List modules = new ArrayList<>(); - private Date startTime; - private volatile boolean running; /** - * Constructs an object that manages a sequence of file level ingest - * modules. It starts the modules, runs files through them, and shuts them - * down when file level ingest is complete. + * Constructs a pipeline of file ingest modules for performing file ingest + * tasks for an ingest job. * - * @param ingestJobPipeline The ingestJobPipeline that owns the pipeline. - * @param moduleTemplates The ingest module templates that define the - * pipeline. + * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. */ FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + super(ingestJobPipeline, moduleTemplates); this.ingestJobPipeline = ingestJobPipeline; - for (IngestModuleTemplate template : moduleTemplates) { - if (template.isFileIngestModuleTemplate()) { - PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); - modules.add(module); - } + } + + @Override + Optional> acceptModuleTemplate(IngestModuleTemplate template) { + Optional> module = Optional.empty(); + if (template.isFileIngestModuleTemplate()) { + FileIngestModule ingestModule = template.createFileIngestModule(); + module = Optional.of(new FileIngestPipelineModule(ingestModule, template.getModuleName())); } + return module; } - /** - * Queries whether or not there are any ingest modules in this pipeline. - * - * @return True or false. - */ - boolean isEmpty() { - return this.modules.isEmpty(); + @Override + void prepareTask(FileIngestTask task) throws IngestTaskPipelineException { } - /** - * Queries whether or not this pipeline is running. - * - * @return True or false. - */ - boolean isRunning() { - return this.running; - } - - /** - * Returns the start up time of this pipeline. - * - * @return The file processing start time, may be null if this pipeline has - * not been started yet. - */ - Date getStartTime() { - return this.startTime; - } - - /** - * Starts up all of the ingest modules in the pipeline. - * - * @return List of start up errors, possibly empty. - */ - synchronized List startUp() { - this.startTime = new Date(); - this.running = true; - List errors = new ArrayList<>(); - for (PipelineModule module : this.modules) { - try { - module.startUp(new IngestJobContext(this.ingestJobPipeline)); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } + @Override + void completeTask(FileIngestTask task) throws IngestTaskPipelineException { + AbstractFile file = null; + try { + file = task.getFile(); + } catch (TskCoreException ex) { + throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS } - return errors; - } - - /** - * Runs a file through the ingest modules in sequential order. - * - * @param task A file level ingest task containing a file to be processed. - * - * @return A list of processing errors, possible empty. - */ - synchronized List process(FileIngestTask task) { - List errors = new ArrayList<>(); - if (!this.ingestJobPipeline.isCancelled()) { - AbstractFile file; - try { - file = task.getFile(); - } catch (TskCoreException ex) { - // In practice, this task would never have been enqueued since the file - // lookup would have failed there. - errors.add(new IngestModuleError("File Ingest Pipeline", ex)); // NON-NLS - FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task); - return errors; + try { + if (!ingestJobPipeline.isCancelled()) { + /* + * Save any updates from the ingest modules to the case + * database. + */ + file.save(); } - for (PipelineModule module : this.modules) { - try { - FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); - this.ingestJobPipeline.setCurrentFileIngestModule(module.getDisplayName(), task.getFile().getName()); - module.process(file); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - if (this.ingestJobPipeline.isCancelled()) { - break; - } - } - - if (!this.ingestJobPipeline.isCancelled()) { - // Save any properties that have not already been saved to the database - try{ - file.save(); - } catch (TskCoreException ex){ - Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data for file " + file.getId(), ex); //NON-NLS - } + } catch (TskCoreException ex) { + throw new IngestTaskPipelineException(String.format("Failed to save updated data for file (file objId = %d)", task.getFileId()), ex); //NON-NLS + } finally { + if (!ingestJobPipeline.isCancelled()) { IngestManager.getInstance().fireFileIngestDone(file); } file.close(); + ingestManager.setIngestTaskProgressCompleted(task); } - FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task); - return errors; } /** - * Shuts down all of the modules in the pipeline. - * - * @return A list of shut down errors, possibly empty. + * A wrapper that adds ingest infrastructure operations to a file ingest + * module. */ - synchronized List shutDown() { - List errors = new ArrayList<>(); - if (this.running == true) { // Don't shut down pipelines that never started - for (PipelineModule module : this.modules) { - try { - module.shutDown(); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - String msg = ex.getMessage(); - // Jython run-time errors don't seem to have a message, but have details in toString. - if (msg == null) { - msg = ex.toString(); - } - MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg); - } - } - } - this.running = false; - return errors; - } - - /** - * This class decorates a file level ingest module with a display name. - */ - private static final class PipelineModule implements FileIngestModule { + static final class FileIngestPipelineModule extends IngestTaskPipeline.PipelineModule { private final FileIngestModule module; - private final String displayName; /** - * Constructs an object that decorates a file level ingest module with a - * display name. + * Constructs a wrapper that adds ingest infrastructure operations to a + * file ingest module. * - * @param module The file level ingest module to be decorated. - * @param displayName The display name. + * + * @param module The module. + * @param displayName The display name of the module. */ - PipelineModule(FileIngestModule module, String displayName) { + FileIngestPipelineModule(FileIngestModule module, String displayName) { + super(module, displayName); this.module = module; - this.displayName = displayName; - } - - /** - * Gets the class name of the decorated ingest module. - * - * @return The class name. - */ - String getClassName() { - return module.getClass().getCanonicalName(); - } - - /** - * Gets the display name of the decorated ingest module. - * - * @return The display name. - */ - String getDisplayName() { - return displayName; } @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); - } - - @Override - public IngestModule.ProcessResult process(AbstractFile file) { - return module.process(file); - } - - @Override - public void shutDown() { - module.shutDown(); + void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { + AbstractFile file = null; + try { + file = task.getFile(); + } catch (TskCoreException ex) { + throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS + } + ingestManager.setIngestTaskProgress(task, getDisplayName()); + ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName()); + ProcessResult result = module.process(file); + if (result == ProcessResult.ERROR) { + throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS + } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index c73dd940d0..decccd4e84 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -416,7 +416,7 @@ public final class IngestJob { Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot); dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot)); if (null == dataSourceModule) { - DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); + DataSourceIngestPipeline.DataSourcePipelineModule module = snapshot.getDataSourceLevelIngestModule(); if (null != module) { dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module); } @@ -500,7 +500,7 @@ public final class IngestJob { public static class DataSourceIngestModuleHandle { private final IngestJobPipeline ingestJobPipeline; - private final DataSourceIngestPipeline.PipelineModule module; + private final DataSourceIngestPipeline.DataSourcePipelineModule module; private final boolean cancelled; /** @@ -511,7 +511,7 @@ public final class IngestJob { * @param ingestJobPipeline The ingestJobPipeline that owns the data source level ingest module. * @param module The data source level ingest module. */ - private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.PipelineModule module) { + private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { this.ingestJobPipeline = ingestJobPipeline; this.module = module; this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index d5ba4c783b..384f5c63c5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2019 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -553,7 +553,7 @@ final class IngestJobPipeline { /* * If the data-source-level ingest pipelines were successfully started, - * start the Start the file-level ingest pipelines (one per file ingest + * start the file-level ingest pipelines (one per pipeline file ingest * thread). */ if (errors.isEmpty()) { @@ -940,7 +940,7 @@ final class IngestJobPipeline { synchronized (this.dataSourceIngestPipelineLock) { if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + errors.addAll(this.currentDataSourceIngestPipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } @@ -1014,7 +1014,7 @@ final class IngestJobPipeline { * Run the file through the pipeline. */ List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); + errors.addAll(pipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors, file); } @@ -1232,9 +1232,9 @@ final class IngestJobPipeline { * * @return The currently running module, may be null. */ - DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { - if (null != this.currentDataSourceIngestPipeline) { - return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); + DataSourceIngestPipeline.DataSourcePipelineModule getCurrentDataSourceIngestModule() { + if (null != currentDataSourceIngestPipeline) { + return (DataSourceIngestPipeline.DataSourcePipelineModule) currentDataSourceIngestPipeline.getCurrentlyRunningModule(); } else { return null; } @@ -1274,7 +1274,7 @@ final class IngestJobPipeline { } } } - + // If a data source had no tasks in progress it may now be complete. checkForStageCompleted(); } @@ -1353,18 +1353,18 @@ final class IngestJobPipeline { logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis", error.getModuleDisplayName()), error.getThrowable()); //NON-NLS } } - + /** * Write ingest module errors to the log. * * @param errors The errors. - * @param file AbstractFile that caused the errors. + * @param file AbstractFile that caused the errors. */ private void logIngestModuleErrors(List errors, AbstractFile file) { for (IngestModuleError error : errors) { logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s, object ID %d", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS } - } + } /** * Gets a snapshot of this jobs state and performance. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 95696378b7..791a45021c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2019 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index c5628d1912..8bfeef57fb 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,9 +25,8 @@ package org.sleuthkit.autopsy.ingest; * ingest job it performs (one for each thread that it is using). * * Autopsy will call startUp() before any data is processed, will pass any data - * to be analyzed into the process() method (FileIngestModule.process() or - * DataSourceIngestModule.process()), and call shutDown() after either all data - * is analyzed or the user has canceled the job. + * to be analyzed into the process() method, and call shutDown() after either + * all data is analyzed or the user has canceled the job. * * Autopsy may use multiple threads to complete an ingest job, but it is * guaranteed that a module instance will always be called from a single thread. @@ -47,25 +46,53 @@ package org.sleuthkit.autopsy.ingest; public interface IngestModule { /** - * A return code for derived class process() methods. + * Invoked by Autopsy to allow an ingest module instance to set up any + * internal data structures and acquire any private resources it will need + * during an ingest job. If the module depends on loading any resources, it + * should do so in this method so that it can throw an exception in the case + * of an error and alert the user. Exceptions that are thrown from startUp() + * are logged and stop processing of the data source. + * + * IMPORTANT: If the module instances must share resources, the modules are + * responsible for synchronizing access to the shared resources and doing + * reference counting as required to release those resources correctly. + * Also, more than one ingest job may be in progress at any given time. This + * must also be taken into consideration when sharing resources between + * module instances. See IngestModuleReferenceCounter. + * + * @param context Provides data and services specific to the ingest job and + * the ingest pipeline of which the module is a part. + * + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ - public enum ProcessResult { - - OK, - ERROR - }; + default void startUp(IngestJobContext context) throws IngestModuleException { + } /** - * A custom exception for the use of ingest modules. + * Invoked by Autopsy when an ingest job is completed (either because the + * data has been analyzed or because the job was cancelled), before the + * ingest module instance is discarded. The module should respond by doing + * things like releasing private resources, submitting final results, and + * posting a final ingest message. + * + * IMPORTANT: If the module instances must share resources, the modules are + * responsible for synchronizing access to the shared resources and doing + * reference counting as required to release those resources correctly. + * Also, more than one ingest job may be in progress at any given time. This + * must also be taken into consideration when sharing resources between + * module instances. See IngestModuleReferenceCounter. + * + */ + default void shutDown() { + } + + /** + * An exception for the use of ingest modules. */ public class IngestModuleException extends Exception { private static final long serialVersionUID = 1L; - @Deprecated - public IngestModuleException() { - } - public IngestModuleException(String message) { super(message); } @@ -73,26 +100,21 @@ public interface IngestModule { public IngestModuleException(String message, Throwable cause) { super(message, cause); } + + @Deprecated + public IngestModuleException() { + } + } /** - * Invoked by Autopsy to allow an ingest module instance to set up any - * internal data structures and acquire any private resources it will need - * during an ingest job. If the module depends on loading any resources, it - * should do so in this method so that it can throw an exception in the case - * of an error and alert the user. Exceptions that are thrown from process() - * and shutDown() are logged, but do not stop processing of the data source. - * - * @param context Provides data and services specific to the ingest job and - * the ingest pipeline of which the module is a part. - * - * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + * A return code for subclass process() methods. */ - void startUp(IngestJobContext context) throws IngestModuleException; + public enum ProcessResult { + + OK, + ERROR + + }; - /** - * TODO: The next time an API change is legal, add a cancel() method and - * remove the "ingest job is canceled" queries from the IngestJobContext - * class. - */ } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java new file mode 100755 index 0000000000..5e918d4ddb --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -0,0 +1,345 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; + +/** + * An abstract superclass for pipelines of ingest modules for a given ingest + * task type. Some examples of ingest task types: data source level ingest + * tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses need to + * implement a specialization of the inner PipelineModule abstract superclass + * for the type of ingest modules that make up the pipeline. + * + * @param The ingest task type. + */ +abstract class IngestTaskPipeline { + + private static final IngestManager ingestManager = IngestManager.getInstance(); + private final IngestJobPipeline ingestJobPipeline; + private final List moduleTemplates; + private final List> modules; + private volatile Date startTime; + private volatile boolean running; + private volatile PipelineModule currentModule; + + /** + * Constructs an instance of an abstract superclass for pipelines of ingest + * modules for a given ingest task type. Some examples of ingest task types: + * data source level ingest tasks, file ingest tasks, data artifact ingest + * tasks, etc. Subclasses need to implement a specialization of the inner + * PipelineModule abstract superclass for the type of ingest modules that + * make up the pipeline. + * + * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. + */ + IngestTaskPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + this.ingestJobPipeline = ingestJobPipeline; + this.moduleTemplates = moduleTemplates; + modules = new ArrayList<>(); + } + + /** + * Indicates whether or not there are any ingest modules in this pipeline. + * + * @return True or false. + */ + boolean isEmpty() { + return modules.isEmpty(); + } + + /** + * Queries whether or not this pipeline is running, i.e., started and not + * shut down. + * + * @return True or false. + */ + boolean isRunning() { + return running; + } + + /** + * Starts up the ingest modules in this pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ + List startUp() { + createIngestModules(moduleTemplates); + return startUpIngestModules(); + } + + /** + * Creates the ingest modules for this pipeline. + * + * @param moduleTemplates The ingest module templates avaialble to this + * pipeline. + */ + private void createIngestModules(List moduleTemplates) { + for (IngestModuleTemplate template : moduleTemplates) { + Optional> module = acceptModuleTemplate(template); + if (module.isPresent()) { + modules.add(module.get()); + } + } + } + + /** + * Determines if the type of ingest module that can be created from a given + * ingest module template should be added to this pipeline. If so, the + * ingest module is created and returned. + * + * @param ingestModuleTemplate The ingest module template to be used or + * ignored, as appropriate to the pipeline type. + * + * @return An Optional that is either empty or contains a newly created and + * wrapped ingest module. + */ + abstract Optional> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate); + + /** + * Starts up the ingest modules in the pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ + private List startUpIngestModules() { + startTime = new Date(); + running = true; + List errors = new ArrayList<>(); + for (PipelineModule module : modules) { + try { + module.startUp(new IngestJobContext(ingestJobPipeline)); + } catch (Throwable ex) { // Catch-all exception firewall + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + } + return errors; + } + + /** + * Returns the start up time of this pipeline. + * + * @return The file processing start time, may be null if this pipeline has + * not been started yet. + */ + Date getStartTime() { + if (startTime == null) { + throw new IllegalStateException("startUp() was not called"); //NON-NLS + } + return new Date(startTime.getTime()); + } + + /** + * Does any preparation required before performing a task. + * + * @param task The task. + * + * @throws IngestTaskPipelineException Thrown if there is an error preparing + * to perform the task. + */ + abstract void prepareTask(T task) throws IngestTaskPipelineException; + + /** + * Performs an ingest task using the ingest modules in this pipeline. + * + * @param task The task. + * + * @return A list of ingest module task processing errors, possibly empty. + */ + List performTask(T task) { + List errors = new ArrayList<>(); + if (!this.ingestJobPipeline.isCancelled()) { + try { + prepareTask(task); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + return errors; + } + for (PipelineModule module : modules) { + try { + currentModule = module; + currentModule.setProcessingStartTime(); + module.performTask(ingestJobPipeline, task); + } catch (Throwable ex) { // Catch-all exception firewall + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (ingestJobPipeline.isCancelled()) { + break; + } + } + } + try { + completeTask(task); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + } + currentModule = null; + return errors; + } + + /** + * Gets the currently running module. + * + * @return The module, possibly null if no module is currently running. + */ + PipelineModule getCurrentlyRunningModule() { + return currentModule; + } + + /** + * Does any clean up required after performing a task. + * + * @param task The task. + * + * @throws IngestTaskPipelineException Thrown if there is an error cleaning + * up after performing the task. + */ + abstract void completeTask(T task) throws IngestTaskPipelineException; + + /** + * Shuts down all of the modules in the pipeline. + * + * @return A list of shut down errors, possibly empty. + */ + List shutDown() { + List errors = new ArrayList<>(); + if (running == true) { + for (PipelineModule module : modules) { + try { + module.shutDown(); + } catch (Throwable ex) { // Catch-all exception firewall + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + String msg = ex.getMessage(); + if (msg == null) { + /* + * Jython run-time errors don't seem to have a message, + * but have details in the string returned by + * toString(). + */ + msg = ex.toString(); + } + MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg); + } + } + } + running = false; + return errors; + } + + /** + * An abstract superclass for a wrapper that adds ingest infrastructure + * operations to an ingest module. + */ + static abstract class PipelineModule implements IngestModule { + + private final IngestModule module; + private final String displayName; + private volatile Date processingStartTime; + + /** + * Constructs an instance of an abstract superclass for a wrapper that + * adds ingest infrastructure operations to an ingest module. + * + * @param module The ingest module to be wrapped. + * @param displayName The display name for the module. + */ + PipelineModule(IngestModule module, String displayName) { + this.module = module; + this.displayName = displayName; + this.processingStartTime = new Date(); + } + + /** + * Gets the class name of the wrapped ingest module. + * + * @return The class name. + */ + String getClassName() { + return module.getClass().getCanonicalName(); + } + + /** + * Gets the display name of the wrapped ingest module. + * + * @return The display name. + */ + String getDisplayName() { + return displayName; + } + + /** + * Sets the processing start time for the wrapped module to the system + * time when this method is called. + */ + void setProcessingStartTime() { + processingStartTime = new Date(); + } + + /** + * Gets the the processing start time for the wrapped module. + * + * @return The start time, will be null if the module has not started + * processing the data source yet. + */ + Date getProcessingStartTime() { + return new Date(processingStartTime.getTime()); + } + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + module.startUp(context); + } + + /** + * Performs an ingest task. + * + * @param ingestJobPipeline The ingest job pipeline that owns the ingest + * module pipeline this module belongs to. + * @param task The task to process. + * + * @throws IngestModuleException Excepton thrown if there is an error + * performing the task. + */ + abstract void performTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException; + + } + + /** + * An exception for the use of ingest task pipelines. + */ + public static class IngestTaskPipelineException extends Exception { + + private static final long serialVersionUID = 1L; + + public IngestTaskPipelineException(String message) { + super(message); + } + + public IngestTaskPipelineException(String message, Throwable cause) { + super(message, cause); + } + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java index 19a0e41c35..dc7eceaad8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java @@ -34,7 +34,7 @@ public final class Snapshot implements Serializable { private final long jobId; private final long jobStartTime; private final long snapShotTime; - transient private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule; + transient private final DataSourceIngestPipeline.DataSourcePipelineModule dataSourceLevelIngestModule; private final boolean fileIngestRunning; private final Date fileIngestStartTime; private final long processedFiles; @@ -48,7 +48,7 @@ public final class Snapshot implements Serializable { * Constructs an object to store basic diagnostic statistics for a data * source ingest job. */ - Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.PipelineModule dataSourceIngestModule, + Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.DataSourcePipelineModule dataSourceIngestModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean jobCancelled, IngestJob.CancellationReason cancellationReason, List cancelledModules, long processedFiles, long estimatedFilesToProcess, @@ -110,7 +110,7 @@ public final class Snapshot implements Serializable { return jobStartTime; } - DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() { + DataSourceIngestPipeline.DataSourcePipelineModule getDataSourceLevelIngestModule() { return this.dataSourceLevelIngestModule; }