From adf56dad3cd3e8fe8d4bccef26c3a64013cb341d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 19 Feb 2021 16:12:42 -0500 Subject: [PATCH 1/7] 7303 only run workers when tabbed pane state changes and domains exist --- .../sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java index c50ab50020..42ccf485c6 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java @@ -80,9 +80,9 @@ final class DomainDetailsPanel extends JPanel { if (selectedTabName == null || !selectedTabName.equals(newTabTitle)) { selectedTabName = newTabTitle; Component selectedComponent = jTabbedPane1.getSelectedComponent(); - if (selectedComponent instanceof DomainArtifactsTabPanel) { + if (!StringUtils.isBlank(domain) && selectedComponent instanceof DomainArtifactsTabPanel) { runDomainWorker((DomainArtifactsTabPanel) selectedComponent, true); - } else if (selectedComponent instanceof MiniTimelinePanel) { + } else if (!StringUtils.isBlank(domain) && selectedComponent instanceof MiniTimelinePanel) { runMiniTimelineWorker((MiniTimelinePanel) selectedComponent, true); } } From e150ba1fb93b658f049d02558d8c950cac5546c0 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 19 Feb 2021 16:42:40 -0500 Subject: [PATCH 2/7] 7303 make reseting of empty artifact type tabs more consistant --- .../sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java | 3 +-- .../sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java index d1328ef68c..cbf38d7453 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java @@ -162,8 +162,7 @@ final class ArtifactsListPanel extends AbstractArtifactListPanel { @ThreadConfined(type = ThreadConfined.ThreadType.AWT) @Override void clearList() { - tableModel.setContents(new ArrayList<>()); - tableModel.fireTableDataChanged(); + addArtifacts(new ArrayList<>()); } /** diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java index 42ccf485c6..496141b091 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java @@ -170,13 +170,13 @@ final class DomainDetailsPanel extends JPanel { @Subscribe void handlePopulateDomainTabsEvent(DiscoveryEventUtils.PopulateDomainTabsEvent populateEvent) { SwingUtilities.invokeLater(() -> { - if (StringUtils.isBlank(populateEvent.getDomain())) { + domain = populateEvent.getDomain(); + if (StringUtils.isBlank(domain)) { resetTabsStatus(); //send fade out event DiscoveryEventUtils.getDiscoveryEventBus().post(new DiscoveryEventUtils.DetailsVisibleEvent(false)); } else { resetTabsStatus(); - domain = populateEvent.getDomain(); Component selectedComponent = jTabbedPane1.getSelectedComponent(); if (selectedComponent instanceof DomainArtifactsTabPanel) { runDomainWorker((DomainArtifactsTabPanel) selectedComponent, false); From 78ff688d2788b72c266631f2728840f9e0eb2ffe Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Feb 2021 16:02:59 -0500 Subject: [PATCH 3/7] 7332 Refactor ingest task pipelines for extensibility --- .../ingest/DataSourceIngestModuleAdapter.java | 5 + .../ingest/DataSourceIngestPipeline.java | 195 ++++---------- .../autopsy/ingest/FileIngestModule.java | 11 +- .../ingest/FileIngestModuleAdapter.java | 5 + .../autopsy/ingest/FileIngestPipeline.java | 250 +++++------------- .../sleuthkit/autopsy/ingest/IngestJob.java | 6 +- .../autopsy/ingest/IngestJobPipeline.java | 22 +- .../autopsy/ingest/IngestManager.java | 4 +- .../autopsy/ingest/IngestModule.java | 86 +++--- .../sleuthkit/autopsy/ingest/Snapshot.java | 6 +- 10 files changed, 199 insertions(+), 391 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java index 6f79817782..a866c278b6 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java @@ -23,7 +23,12 @@ import org.sleuthkit.datamodel.Content; /** * An adapter that provides a no-op implementation of the startUp() method for * data source ingest modules. + * + * @deprecated As of Java 8, interfaces can have default methods. + * DataSourceIngestModule now provides default no-op versions of startUp() + * and shutDown(). */ +@Deprecated public abstract class DataSourceIngestModuleAdapter implements DataSourceIngestModule { @Override diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index c22d63348c..0ad1a3e60a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2011-2016 Basis Technology Corp. + * + * Copyright 2012-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,184 +18,83 @@ */ package org.sleuthkit.autopsy.ingest; -import java.util.ArrayList; -import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source level ingest modules for an - * ingestJobPipeline. It starts the modules, runs data sources through them, and - * shuts them down when data source level ingest is complete. - *

- * This class is thread-safe. + * A pipeline of data source level ingest modules for performing data source + * level ingest tasks for an ingest job. */ -final class DataSourceIngestPipeline { +final class DataSourceIngestPipeline extends IngestTaskPipeline { - private static final IngestManager ingestManager = IngestManager.getInstance(); private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName()); - private final IngestJobPipeline ingestJobPipeline; - private final List modules = new ArrayList<>(); - private volatile PipelineModule currentModule; + private static final IngestManager ingestManager = IngestManager.getInstance(); /** - * Constructs an object that manages a sequence of data source level ingest - * modules. It starts the modules, runs data sources through them, and shuts - * them down when data source level ingest is complete. + * Constructs a pipeline of data source level ingest modules for performing + * data source level ingest tasks for an ingest job. * - * @param ingestJobPipeline The ingestJobPipeline that owns this pipeline. - * @param moduleTemplates Templates for the creating the ingest modules that - * make up this pipeline. + * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. */ DataSourceIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { - this.ingestJobPipeline = ingestJobPipeline; - for (IngestModuleTemplate template : moduleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); - modules.add(module); - } - } + super(ingestJobPipeline, moduleTemplates); } - /** - * Indicates whether or not there are any ingest modules in this pipeline. - * - * @return True or false. - */ - boolean isEmpty() { - return modules.isEmpty(); + @Override + Optional> acceptModuleTemplate(IngestModuleTemplate template) { + Optional> module = Optional.empty(); + if (template.isDataSourceIngestModuleTemplate()) { + DataSourceIngestModule ingestModule = template.createDataSourceIngestModule(); + module = Optional.of(new DataSourcePipelineModule(ingestModule, template.getModuleName())); + } + return module; } - /** - * Starts up the ingest modules in this pipeline. - * - * @return A list of ingest module startup errors, possibly empty. - */ - synchronized List startUp() { - List errors = new ArrayList<>(); - for (PipelineModule module : modules) { - try { - module.startUp(new IngestJobContext(this.ingestJobPipeline)); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - } - return errors; + @Override + void prepareTask(DataSourceIngestTask task) { } - /** - * Runs a data source through the ingest modules in sequential order. - * - * @param task A data source level ingest task containing a data source to - * be processed. - * - * @return A list of processing errors, possible empty. - */ - synchronized List process(DataSourceIngestTask task) { - List errors = new ArrayList<>(); - if (!this.ingestJobPipeline.isCancelled()) { - Content dataSource = task.getDataSource(); - for (PipelineModule module : modules) { - try { - this.currentModule = module; - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.displayName", - module.getDisplayName(), dataSource.getName()); - this.ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(displayName); - this.ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate(); - DataSourceIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); - logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) starting", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS - module.process(dataSource, new DataSourceIngestModuleProgress(this.ingestJobPipeline)); - logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) finished", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - if (this.ingestJobPipeline.isCancelled()) { - break; - } else if (this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) { - this.ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(currentModule.getDisplayName()); - } - } - } - this.currentModule = null; + @Override + void completeTask(DataSourceIngestTask task) { ingestManager.setIngestTaskProgressCompleted(task); - return errors; } /** - * Gets the currently running module. - * - * @return The module, possibly null if no module is currently running. + * A wrapper that adds ingest infrastructure operations to a data source + * level ingest module. */ - PipelineModule getCurrentlyRunningModule() { - return this.currentModule; - } - - /** - * This class decorates a data source level ingest module with a display - * name and a processing start time. - */ - static class PipelineModule implements DataSourceIngestModule { + static final class DataSourcePipelineModule extends IngestTaskPipeline.PipelineModule { private final DataSourceIngestModule module; - private final String displayName; - private volatile Date processingStartTime; /** - * Constructs an object that decorates a data source level ingest module - * with a display name and a processing start time. - * - * @param module The data source level ingest module to be - * decorated. - * @param displayName The display name. + * Constructs a wrapper that adds ingest infrastructure operations to a + * data source level ingest module. */ - PipelineModule(DataSourceIngestModule module, String displayName) { + DataSourcePipelineModule(DataSourceIngestModule module, String displayName) { + super(module, displayName); this.module = module; - this.displayName = displayName; - this.processingStartTime = new Date(); - } - - /** - * Gets the class name of the decorated ingest module. - * - * @return The class name. - */ - String getClassName() { - return this.module.getClass().getCanonicalName(); - } - - /** - * Gets the display of the decorated ingest module. - * - * @return The display name. - */ - String getDisplayName() { - return this.displayName; - } - - /** - * Gets the time the decorated ingest module started processing the data - * source. - * - * @return The start time, will be null if the module has not started - * processing the data source yet. - */ - Date getProcessingStartTime() { - return this.processingStartTime; } @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - this.module.startUp(context); - } - - @Override - public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - this.processingStartTime = new Date(); - return this.module.process(dataSource, statusHelper); + void performTask(IngestJobPipeline ingestJobPipeline, DataSourceIngestTask task) throws IngestModuleException { + Content dataSource = task.getDataSource(); + String progressBarDisplayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", getDisplayName(), dataSource.getName()); + ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(progressBarDisplayName); + ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate(); + ingestManager.setIngestTaskProgress(task, getDisplayName()); + logger.log(Level.INFO, "{0} analysis of {1} starting", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS + module.process(dataSource, new DataSourceIngestModuleProgress(ingestJobPipeline)); + logger.log(Level.INFO, "{0} analysis of {1} finished", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS + if (ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) { + ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(getDisplayName()); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java index 215c9e0bcb..60355e10f8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,13 +36,4 @@ public interface FileIngestModule extends IngestModule { */ ProcessResult process(AbstractFile file); - /** - * Invoked by Autopsy when an ingest job is completed (either because the - * data has been analyzed or because the job was canceled - check - * IngestJobContext.fileIngestIsCancelled()), before the ingest module - * instance is discarded. The module should respond by doing things like - * releasing private resources, submitting final results, and posting a - * final ingest message. - */ - void shutDown(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java index 13811e6eb4..6fbdf529fd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java @@ -23,7 +23,12 @@ import org.sleuthkit.datamodel.AbstractFile; /** * An adapter that provides no-op implementations of the startUp() and * shutDown() methods for file ingest modules. + * + * @deprecated As of Java 8, interfaces can have default methods. + * FileIngestModule now provides default no-op versions of startUp() and + * shutDown(). */ +@Deprecated public abstract class FileIngestModuleAdapter implements FileIngestModule { @Override diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 2f6604e415..12dc2e28b2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2014-2015 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,223 +18,109 @@ */ package org.sleuthkit.autopsy.ingest; -import java.util.ArrayList; -import java.util.Date; import java.util.List; -import java.util.logging.Level; - -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import java.util.Optional; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * This class manages a sequence of file level ingest modules for an - * ingest job pipeline. It starts the modules, runs files through them, and shuts them - * down when file level ingest is complete. - *

- * This class is thread-safe. + * A pipeline of file ingest modules for performing file ingest tasks for an + * ingest job. */ -final class FileIngestPipeline { +final class FileIngestPipeline extends IngestTaskPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private final IngestJobPipeline ingestJobPipeline; - private final List modules = new ArrayList<>(); - private Date startTime; - private volatile boolean running; /** - * Constructs an object that manages a sequence of file level ingest - * modules. It starts the modules, runs files through them, and shuts them - * down when file level ingest is complete. + * Constructs a pipeline of file ingest modules for performing file ingest + * tasks for an ingest job. * - * @param ingestJobPipeline The ingestJobPipeline that owns the pipeline. - * @param moduleTemplates The ingest module templates that define the - * pipeline. + * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. */ FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + super(ingestJobPipeline, moduleTemplates); this.ingestJobPipeline = ingestJobPipeline; - for (IngestModuleTemplate template : moduleTemplates) { - if (template.isFileIngestModuleTemplate()) { - PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); - modules.add(module); - } + } + + @Override + Optional> acceptModuleTemplate(IngestModuleTemplate template) { + Optional> module = Optional.empty(); + if (template.isFileIngestModuleTemplate()) { + FileIngestModule ingestModule = template.createFileIngestModule(); + module = Optional.of(new FileIngestPipelineModule(ingestModule, template.getModuleName())); } + return module; } - /** - * Queries whether or not there are any ingest modules in this pipeline. - * - * @return True or false. - */ - boolean isEmpty() { - return this.modules.isEmpty(); + @Override + void prepareTask(FileIngestTask task) throws IngestTaskPipelineException { } - /** - * Queries whether or not this pipeline is running. - * - * @return True or false. - */ - boolean isRunning() { - return this.running; - } - - /** - * Returns the start up time of this pipeline. - * - * @return The file processing start time, may be null if this pipeline has - * not been started yet. - */ - Date getStartTime() { - return this.startTime; - } - - /** - * Starts up all of the ingest modules in the pipeline. - * - * @return List of start up errors, possibly empty. - */ - synchronized List startUp() { - this.startTime = new Date(); - this.running = true; - List errors = new ArrayList<>(); - for (PipelineModule module : this.modules) { - try { - module.startUp(new IngestJobContext(this.ingestJobPipeline)); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } + @Override + void completeTask(FileIngestTask task) throws IngestTaskPipelineException { + AbstractFile file = null; + try { + file = task.getFile(); + } catch (TskCoreException ex) { + throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS } - return errors; - } - - /** - * Runs a file through the ingest modules in sequential order. - * - * @param task A file level ingest task containing a file to be processed. - * - * @return A list of processing errors, possible empty. - */ - synchronized List process(FileIngestTask task) { - List errors = new ArrayList<>(); - if (!this.ingestJobPipeline.isCancelled()) { - AbstractFile file; - try { - file = task.getFile(); - } catch (TskCoreException ex) { - // In practice, this task would never have been enqueued since the file - // lookup would have failed there. - errors.add(new IngestModuleError("File Ingest Pipeline", ex)); // NON-NLS - FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task); - return errors; + try { + if (!ingestJobPipeline.isCancelled()) { + /* + * Save any updates from the ingest modules to the case + * database. + */ + file.save(); } - for (PipelineModule module : this.modules) { - try { - FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); - this.ingestJobPipeline.setCurrentFileIngestModule(module.getDisplayName(), task.getFile().getName()); - module.process(file); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } - if (this.ingestJobPipeline.isCancelled()) { - break; - } - } - - if (!this.ingestJobPipeline.isCancelled()) { - // Save any properties that have not already been saved to the database - try{ - file.save(); - } catch (TskCoreException ex){ - Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data for file " + file.getId(), ex); //NON-NLS - } + } catch (TskCoreException ex) { + throw new IngestTaskPipelineException(String.format("Failed to save updated data for file (file objId = %d)", task.getFileId()), ex); //NON-NLS + } finally { + if (!ingestJobPipeline.isCancelled()) { IngestManager.getInstance().fireFileIngestDone(file); } file.close(); + ingestManager.setIngestTaskProgressCompleted(task); } - FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task); - return errors; } /** - * Shuts down all of the modules in the pipeline. - * - * @return A list of shut down errors, possibly empty. + * A wrapper that adds ingest infrastructure operations to a file ingest + * module. */ - synchronized List shutDown() { - List errors = new ArrayList<>(); - if (this.running == true) { // Don't shut down pipelines that never started - for (PipelineModule module : this.modules) { - try { - module.shutDown(); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); - String msg = ex.getMessage(); - // Jython run-time errors don't seem to have a message, but have details in toString. - if (msg == null) { - msg = ex.toString(); - } - MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg); - } - } - } - this.running = false; - return errors; - } - - /** - * This class decorates a file level ingest module with a display name. - */ - private static final class PipelineModule implements FileIngestModule { + static final class FileIngestPipelineModule extends IngestTaskPipeline.PipelineModule { private final FileIngestModule module; - private final String displayName; /** - * Constructs an object that decorates a file level ingest module with a - * display name. + * Constructs a wrapper that adds ingest infrastructure operations to a + * file ingest module. * - * @param module The file level ingest module to be decorated. - * @param displayName The display name. + * + * @param module The module. + * @param displayName The display name of the module. */ - PipelineModule(FileIngestModule module, String displayName) { + FileIngestPipelineModule(FileIngestModule module, String displayName) { + super(module, displayName); this.module = module; - this.displayName = displayName; - } - - /** - * Gets the class name of the decorated ingest module. - * - * @return The class name. - */ - String getClassName() { - return module.getClass().getCanonicalName(); - } - - /** - * Gets the display name of the decorated ingest module. - * - * @return The display name. - */ - String getDisplayName() { - return displayName; } @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - module.startUp(context); - } - - @Override - public IngestModule.ProcessResult process(AbstractFile file) { - return module.process(file); - } - - @Override - public void shutDown() { - module.shutDown(); + void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { + AbstractFile file = null; + try { + file = task.getFile(); + } catch (TskCoreException ex) { + throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS + } + ingestManager.setIngestTaskProgress(task, getDisplayName()); + ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName()); + ProcessResult result = module.process(file); + if (result == ProcessResult.ERROR) { + throw new IngestModuleException(String.format("%s experienced an error analyzing file (file objId = %d)", getDisplayName(), task.getFileId())); //NON-NLS + } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index c73dd940d0..decccd4e84 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -416,7 +416,7 @@ public final class IngestJob { Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot); dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot)); if (null == dataSourceModule) { - DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); + DataSourceIngestPipeline.DataSourcePipelineModule module = snapshot.getDataSourceLevelIngestModule(); if (null != module) { dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module); } @@ -500,7 +500,7 @@ public final class IngestJob { public static class DataSourceIngestModuleHandle { private final IngestJobPipeline ingestJobPipeline; - private final DataSourceIngestPipeline.PipelineModule module; + private final DataSourceIngestPipeline.DataSourcePipelineModule module; private final boolean cancelled; /** @@ -511,7 +511,7 @@ public final class IngestJob { * @param ingestJobPipeline The ingestJobPipeline that owns the data source level ingest module. * @param module The data source level ingest module. */ - private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.PipelineModule module) { + private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) { this.ingestJobPipeline = ingestJobPipeline; this.module = module; this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index d5ba4c783b..384f5c63c5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2019 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -553,7 +553,7 @@ final class IngestJobPipeline { /* * If the data-source-level ingest pipelines were successfully started, - * start the Start the file-level ingest pipelines (one per file ingest + * start the file-level ingest pipelines (one per pipeline file ingest * thread). */ if (errors.isEmpty()) { @@ -940,7 +940,7 @@ final class IngestJobPipeline { synchronized (this.dataSourceIngestPipelineLock) { if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + errors.addAll(this.currentDataSourceIngestPipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } @@ -1014,7 +1014,7 @@ final class IngestJobPipeline { * Run the file through the pipeline. */ List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); + errors.addAll(pipeline.performTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors, file); } @@ -1232,9 +1232,9 @@ final class IngestJobPipeline { * * @return The currently running module, may be null. */ - DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { - if (null != this.currentDataSourceIngestPipeline) { - return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); + DataSourceIngestPipeline.DataSourcePipelineModule getCurrentDataSourceIngestModule() { + if (null != currentDataSourceIngestPipeline) { + return (DataSourceIngestPipeline.DataSourcePipelineModule) currentDataSourceIngestPipeline.getCurrentlyRunningModule(); } else { return null; } @@ -1274,7 +1274,7 @@ final class IngestJobPipeline { } } } - + // If a data source had no tasks in progress it may now be complete. checkForStageCompleted(); } @@ -1353,18 +1353,18 @@ final class IngestJobPipeline { logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis", error.getModuleDisplayName()), error.getThrowable()); //NON-NLS } } - + /** * Write ingest module errors to the log. * * @param errors The errors. - * @param file AbstractFile that caused the errors. + * @param file AbstractFile that caused the errors. */ private void logIngestModuleErrors(List errors, AbstractFile file) { for (IngestModuleError error : errors) { logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s, object ID %d", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS } - } + } /** * Gets a snapshot of this jobs state and performance. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 95696378b7..d8a8480776 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2019 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -804,7 +804,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { * * @param task The file level ingest job task that was completed. */ - void setIngestTaskProgressCompleted(FileIngestTask task) { + void setIngestTaskProgressCompleted(IngestTask task) { IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId()); ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index c5628d1912..10114ac288 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,9 +25,8 @@ package org.sleuthkit.autopsy.ingest; * ingest job it performs (one for each thread that it is using). * * Autopsy will call startUp() before any data is processed, will pass any data - * to be analyzed into the process() method (FileIngestModule.process() or - * DataSourceIngestModule.process()), and call shutDown() after either all data - * is analyzed or the user has canceled the job. + * to be analyzed into the process() method, and call shutDown() after either + * all data is analyzed or the user has canceled the job. * * Autopsy may use multiple threads to complete an ingest job, but it is * guaranteed that a module instance will always be called from a single thread. @@ -47,25 +46,53 @@ package org.sleuthkit.autopsy.ingest; public interface IngestModule { /** - * A return code for derived class process() methods. + * Invoked by Autopsy to allow an ingest module instance to set up any + * internal data structures and acquire any private resources it will need + * during an ingest job. If the module depends on loading any resources, it + * should do so in this method so that it can throw an exception in the case + * of an error and alert the user. Exceptions that are thrown from startUp() + * are logged and stop processing of the data source. + * + * IMPORTANT: If the module instances must share resources, the modules are + * responsible for synchronizing access to the shared resources and doing + * reference counting as required to release those resources correctly. + * Also, more than one ingest job may be in progress at any given time. This + * must also be taken into consideration when sharing resources between + * module instances. See IngestModuleReferenceCounter. + * + * @param context Provides data and services specific to the ingest job and + * the ingest pipeline of which the module is a part. + * + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ - public enum ProcessResult { - - OK, - ERROR - }; + default void startUp(IngestJobContext context) throws IngestModuleException { + } /** - * A custom exception for the use of ingest modules. + * Invoked by Autopsy when an ingest job is completed (either because the + * data has been analyzed or because the job was cancelled), before the + * ingest module instance is discarded. The module should respond by doing + * things like releasing private resources, submitting final results, and + * posting a final ingest message. + * + * IMPORTANT: If the module instances must share resources, the modules are + * responsible for synchronizing access to the shared resources and doing + * reference counting as required to release those resources correctly. + * Also, more than one ingest job may be in progress at any given time. This + * must also be taken into consideration when sharing resources between + * module instances. See IngestModuleReferenceCounter. + * + */ + default void shutDown() { + } + + /** + * An exception for the use of ingest modules. */ public class IngestModuleException extends Exception { private static final long serialVersionUID = 1L; - @Deprecated - public IngestModuleException() { - } - public IngestModuleException(String message) { super(message); } @@ -73,26 +100,21 @@ public interface IngestModule { public IngestModuleException(String message, Throwable cause) { super(message, cause); } + + @Deprecated + public IngestModuleException() { + } + } /** - * Invoked by Autopsy to allow an ingest module instance to set up any - * internal data structures and acquire any private resources it will need - * during an ingest job. If the module depends on loading any resources, it - * should do so in this method so that it can throw an exception in the case - * of an error and alert the user. Exceptions that are thrown from process() - * and shutDown() are logged, but do not stop processing of the data source. - * - * @param context Provides data and services specific to the ingest job and - * the ingest pipeline of which the module is a part. - * - * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + * A return code for subclass process() methods. */ - void startUp(IngestJobContext context) throws IngestModuleException; + public enum ProcessResult { + + OK, + ERROR + + }; - /** - * TODO: The next time an API change is legal, add a cancel() method and - * remove the "ingest job is canceled" queries from the IngestJobContext - * class. - */ } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java index 19a0e41c35..dc7eceaad8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java @@ -34,7 +34,7 @@ public final class Snapshot implements Serializable { private final long jobId; private final long jobStartTime; private final long snapShotTime; - transient private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule; + transient private final DataSourceIngestPipeline.DataSourcePipelineModule dataSourceLevelIngestModule; private final boolean fileIngestRunning; private final Date fileIngestStartTime; private final long processedFiles; @@ -48,7 +48,7 @@ public final class Snapshot implements Serializable { * Constructs an object to store basic diagnostic statistics for a data * source ingest job. */ - Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.PipelineModule dataSourceIngestModule, + Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.DataSourcePipelineModule dataSourceIngestModule, boolean fileIngestRunning, Date fileIngestStartTime, boolean jobCancelled, IngestJob.CancellationReason cancellationReason, List cancelledModules, long processedFiles, long estimatedFilesToProcess, @@ -110,7 +110,7 @@ public final class Snapshot implements Serializable { return jobStartTime; } - DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() { + DataSourceIngestPipeline.DataSourcePipelineModule getDataSourceLevelIngestModule() { return this.dataSourceLevelIngestModule; } From 6838641d106284c344cb8000b8fb054b34202b80 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 22 Feb 2021 16:03:29 -0500 Subject: [PATCH 4/7] 7332 Refactor ingest task pipelines for extensibility --- .../ingest/DataArtifactIngestModule.java | 40 ++ .../autopsy/ingest/IngestTaskPipeline.java | 353 ++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java new file mode 100755 index 0000000000..b5677242cb --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java @@ -0,0 +1,40 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; +import org.sleuthkit.datamodel.DataArtifact; + +/** + * Intercace that must be implemented by all ingest modules that process + * artifacts. + */ +public interface DataArtifactIngestModule { + + /** + * Processes a data artifact. + * + * @param artifact The artifact to process. + * + * @throws IngestModuleException Exception is thrown if there is an error + * while processing the data artifact. + */ + void process(DataArtifact artifact) throws IngestModuleException; + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java new file mode 100755 index 0000000000..aaa430b05e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -0,0 +1,353 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; + +/** + * An abstract superclass for pipelines of ingest modules for a given ingest + * task type. Some examples of ingest task types: data source level ingest + * tasks, file ingest tasks, data artifacts ingest tasks, etc. Subclasses need + * to implement a specialization of the inner PipelineModule decorator abstartc + * superclass for they type of ingest modules that make up the pipeline. + * + * @param The ingest task type. + */ +abstract class IngestTaskPipeline { + + private static final IngestManager ingestManager = IngestManager.getInstance(); + private final IngestJobPipeline ingestJobPipeline; + private final List moduleTemplates; + private final List> modules; + private volatile Date startTime; + private volatile boolean running; + private volatile PipelineModule currentModule; + + /** + * Constructs an instance of an abstract superclass for pipelines of ingest + * modules for a given ingest task type. Some examples of ingest task types: + * data source level ingest tasks, file ingest tasks, data artifacts ingest + * tasks, etc. Subclasses need to implement a specialization of the inner + * PipelineModule decorator abstartc superclass for they type of ingest + * modules that make up the pipeline. + * + * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. + */ + IngestTaskPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + this.ingestJobPipeline = ingestJobPipeline; + this.moduleTemplates = moduleTemplates; + modules = new ArrayList<>(); + } + + /** + * Indicates whether or not there are any ingest modules in this pipeline. + * + * @return True or false. + */ + boolean isEmpty() { + return modules.isEmpty(); + } + + /** + * Queries whether or not this pipeline is running, i.e., started and not + * shut down. + * + * @return True or false. + */ + boolean isRunning() { + return running; + } + + /** + * Starts up the ingest modules in this pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ + List startUp() { + createIngestModules(moduleTemplates); + return startUpIngestModules(); + } + + /** + * Creates the ingest modules for this pipeline. + * + * @param moduleTemplates The ingest module templates avaialble to this + * pipeline. + */ + private void createIngestModules(List moduleTemplates) { + for (IngestModuleTemplate template : moduleTemplates) { + Optional> module = acceptModuleTemplate(template); + if (module.isPresent()) { + modules.add(module.get()); + } + } + } + + /** + * Determines if the type of ingest module that can be created from a given + * ingest module template should be added to this pipeline. If so, the + * ingest module is created and returned. + * + * @param ingestModuleTemplate The ingest module template to be used or + * ignored, as appropriate to the pipeline type. + * + * @return An Optional that is either empty or contains a newly created and + * decorated ingest module. + */ + abstract Optional> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate); + + /** + * Starts up the ingest modules in the pipeline. + * + * @return A list of ingest module startup errors, possibly empty. + */ + private List startUpIngestModules() { + startTime = new Date(); + running = true; + List errors = new ArrayList<>(); + for (PipelineModule module : modules) { + try { + module.startUp(new IngestJobContext(ingestJobPipeline)); + } catch (Throwable ex) { // Catch-all exception firewall + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + } + return errors; + } + + /** + * Returns the start up time of this pipeline. + * + * @return The file processing start time, may be null if this pipeline has + * not been started yet. + */ + Date getStartTime() { + if (startTime == null) { + throw new IllegalStateException("startUp() was not called"); //NON-NLS + } + return new Date(startTime.getTime()); + } + + /** + * Does any preparation required before performing the a task. + * + * @param task The task. + * + * @throws IngestTaskPipelineException Thrown if there is an error preparing + * to perform the task. + */ + abstract void prepareTask(T task) throws IngestTaskPipelineException; + + /** + * Performs an ingest task using the ingest modules in this pipeline. + * + * @param task The task. + * + * @return A list of ingest module task processing errors, possibly empty. + */ + List performTask(T task) { + List errors = new ArrayList<>(); + if (!this.ingestJobPipeline.isCancelled()) { + try { + prepareTask(task); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + return errors; + } + for (PipelineModule module : modules) { + try { + currentModule = module; + currentModule.setProcessingStartTime(); + module.performTask(ingestJobPipeline, task); + } catch (Throwable ex) { // Catch-all exception firewall + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (ingestJobPipeline.isCancelled()) { + break; + } + } + } + try { + completeTask(task); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + } + ingestManager.setIngestTaskProgressCompleted(task); + currentModule = null; + return errors; + } + + /** + * Gets the currently running module. + * + * @return The module, possibly null if no module is currently running. + */ + PipelineModule getCurrentlyRunningModule() { + return currentModule; + } + + /** + * Does any clean up required after performing the current task. + * + * @param task The current task. + * + * @throws IngestTaskPipelineException Thrown if there is an error cleaning + * up after performing the task. + */ + abstract void completeTask(T task) throws IngestTaskPipelineException; + + /** + * Shuts down all of the modules in the pipeline. + * + * @return A list of shut down errors, possibly empty. + */ + synchronized List shutDown() { + List errors = new ArrayList<>(); + if (running == true) { + for (PipelineModule module : modules) { + try { + module.shutDown(); + } catch (Throwable ex) { // Catch-all exception firewall + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + String msg = ex.getMessage(); + if (msg == null) { + /* + * Jython run-time errors don't seem to have a message, + * but have details in toString(). + */ + msg = ex.toString(); + } + MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg); + } + } + } + running = false; + return errors; + } + + /** + * An abstract superclass for a wrapper that adds ingest infrastructure + * operations to an ingest module. + */ + static abstract class PipelineModule implements IngestModule { + + private final IngestModule module; + private final String displayName; + + /* + * This field is intended to be written to in an ingest thread and read + * from in an ingest snaphot thread. + */ + private volatile Date processingStartTime; + + /** + * Constructs an instance of an abstract superclass for a wrapper that + * adds ingest infrastructure operations to an ingest module. + * + * @param module The ingest module to be decorated. + * @param displayName The display name for the module. + */ + PipelineModule(IngestModule module, String displayName) { + this.module = module; + this.displayName = displayName; + this.processingStartTime = new Date(); + } + + /** + * Gets the class name of the wrapped ingest module. + * + * @return The class name. + */ + String getClassName() { + return module.getClass().getCanonicalName(); + } + + /** + * Gets the display name of the wrapped ingest module. + * + * @return The display name. + */ + String getDisplayName() { + return displayName; + } + + /** + * Sets the processing start time for the wrapped module to the system + * time when this method is called. + */ + void setProcessingStartTime() { + processingStartTime = new Date(); + } + + /** + * Gets the the processing start time for the wrapped module, as set by + * + * @return The start time, will be null if the module has not started + * processing the data source yet. + */ + Date getProcessingStartTime() { + if (processingStartTime == null) { + throw new IllegalStateException("setProcessingStartTime() was not called"); //NON-NLS + } + return new Date(processingStartTime.getTime()); + } + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + module.startUp(context); + } + + /** + * Processes an ingest task. + * + * @param ingestJobPipeline The ingest job pipeline that owns the ingest + * module pipeline this module belongs to. + * @param task The task to process. + * + * @throws IngestModuleException Excepton thrown if there is an error + * performing the task. + */ + abstract void performTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException; + + } + + /** + * An exception for the use of ingest task pipelines. + */ + public static class IngestTaskPipelineException extends Exception { + + private static final long serialVersionUID = 1L; + + public IngestTaskPipelineException(String message) { + super(message); + } + + public IngestTaskPipelineException(String message, Throwable cause) { + super(message, cause); + } + } + +} From 84f30369510fa523cf3225ccc695827846c6568e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 23 Feb 2021 11:58:57 -0500 Subject: [PATCH 5/7] 7332 Refactor ingest task pipelines for extensibility --- .../autopsy/ingest/DataSourceIngestModuleAdapter.java | 8 ++++---- .../sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java index a866c278b6..83b5a1457f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java @@ -24,11 +24,11 @@ import org.sleuthkit.datamodel.Content; * An adapter that provides a no-op implementation of the startUp() method for * data source ingest modules. * - * @deprecated As of Java 8, interfaces can have default methods. - * DataSourceIngestModule now provides default no-op versions of startUp() - * and shutDown(). + * NOTE: As of Java 8, interfaces can have default methods. + * DataSourceIngestModule now provides default no-op versions of startUp() and + * shutDown(). This class is no longer needed and can be deprecated when + * convenient. */ -@Deprecated public abstract class DataSourceIngestModuleAdapter implements DataSourceIngestModule { @Override diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java index 6fbdf529fd..9f733308c8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java @@ -24,11 +24,10 @@ import org.sleuthkit.datamodel.AbstractFile; * An adapter that provides no-op implementations of the startUp() and * shutDown() methods for file ingest modules. * - * @deprecated As of Java 8, interfaces can have default methods. - * FileIngestModule now provides default no-op versions of startUp() and - * shutDown(). + * NOTE: As of Java 8, interfaces can have default methods. FileIngestModule now + * provides default no-op versions of startUp() and shutDown(). This class is no + * longer needed and can be deprecated when convenient. */ -@Deprecated public abstract class FileIngestModuleAdapter implements FileIngestModule { @Override From 62de904bb801793c3a02cad22fc573e9e3484887 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 23 Feb 2021 12:03:22 -0500 Subject: [PATCH 6/7] 7332 Refactor ingest task pipelines for extensibility --- .../ingest/DataArtifactIngestModule.java | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java deleted file mode 100755 index b5677242cb..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2021 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; -import org.sleuthkit.datamodel.DataArtifact; - -/** - * Intercace that must be implemented by all ingest modules that process - * artifacts. - */ -public interface DataArtifactIngestModule { - - /** - * Processes a data artifact. - * - * @param artifact The artifact to process. - * - * @throws IngestModuleException Exception is thrown if there is an error - * while processing the data artifact. - */ - void process(DataArtifact artifact) throws IngestModuleException; - -} From b03844db67c9fac67f49705ade4e78a242eafb3e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 23 Feb 2021 13:05:58 -0500 Subject: [PATCH 7/7] 7332 Refactor ingest task pipelines for extensibility --- .../ingest/DataSourceIngestPipeline.java | 9 +++-- .../autopsy/ingest/FileIngestPipeline.java | 2 +- .../autopsy/ingest/IngestManager.java | 2 +- .../autopsy/ingest/IngestModule.java | 2 +- .../autopsy/ingest/IngestTaskPipeline.java | 40 ++++++++----------- 5 files changed, 25 insertions(+), 30 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 0ad1a3e60a..6c551da602 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2021 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -90,11 +90,14 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline { ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName()); ProcessResult result = module.process(file); if (result == ProcessResult.ERROR) { - throw new IngestModuleException(String.format("%s experienced an error analyzing file (file objId = %d)", getDisplayName(), task.getFileId())); //NON-NLS + throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index d8a8480776..791a45021c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -804,7 +804,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { * * @param task The file level ingest job task that was completed. */ - void setIngestTaskProgressCompleted(IngestTask task) { + void setIngestTaskProgressCompleted(FileIngestTask task) { IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId()); ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index 10114ac288..8bfeef57fb 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2021 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java index aaa430b05e..5e918d4ddb 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -28,9 +28,9 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; /** * An abstract superclass for pipelines of ingest modules for a given ingest * task type. Some examples of ingest task types: data source level ingest - * tasks, file ingest tasks, data artifacts ingest tasks, etc. Subclasses need - * to implement a specialization of the inner PipelineModule decorator abstartc - * superclass for they type of ingest modules that make up the pipeline. + * tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses need to + * implement a specialization of the inner PipelineModule abstract superclass + * for the type of ingest modules that make up the pipeline. * * @param The ingest task type. */ @@ -47,10 +47,10 @@ abstract class IngestTaskPipeline { /** * Constructs an instance of an abstract superclass for pipelines of ingest * modules for a given ingest task type. Some examples of ingest task types: - * data source level ingest tasks, file ingest tasks, data artifacts ingest + * data source level ingest tasks, file ingest tasks, data artifact ingest * tasks, etc. Subclasses need to implement a specialization of the inner - * PipelineModule decorator abstartc superclass for they type of ingest - * modules that make up the pipeline. + * PipelineModule abstract superclass for the type of ingest modules that + * make up the pipeline. * * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. * @param moduleTemplates The ingest module templates that define this @@ -115,7 +115,7 @@ abstract class IngestTaskPipeline { * ignored, as appropriate to the pipeline type. * * @return An Optional that is either empty or contains a newly created and - * decorated ingest module. + * wrapped ingest module. */ abstract Optional> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate); @@ -152,7 +152,7 @@ abstract class IngestTaskPipeline { } /** - * Does any preparation required before performing the a task. + * Does any preparation required before performing a task. * * @param task The task. * @@ -195,7 +195,6 @@ abstract class IngestTaskPipeline { } catch (IngestTaskPipelineException ex) { errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS } - ingestManager.setIngestTaskProgressCompleted(task); currentModule = null; return errors; } @@ -210,9 +209,9 @@ abstract class IngestTaskPipeline { } /** - * Does any clean up required after performing the current task. + * Does any clean up required after performing a task. * - * @param task The current task. + * @param task The task. * * @throws IngestTaskPipelineException Thrown if there is an error cleaning * up after performing the task. @@ -224,7 +223,7 @@ abstract class IngestTaskPipeline { * * @return A list of shut down errors, possibly empty. */ - synchronized List shutDown() { + List shutDown() { List errors = new ArrayList<>(); if (running == true) { for (PipelineModule module : modules) { @@ -236,7 +235,8 @@ abstract class IngestTaskPipeline { if (msg == null) { /* * Jython run-time errors don't seem to have a message, - * but have details in toString(). + * but have details in the string returned by + * toString(). */ msg = ex.toString(); } @@ -256,18 +256,13 @@ abstract class IngestTaskPipeline { private final IngestModule module; private final String displayName; - - /* - * This field is intended to be written to in an ingest thread and read - * from in an ingest snaphot thread. - */ private volatile Date processingStartTime; /** * Constructs an instance of an abstract superclass for a wrapper that * adds ingest infrastructure operations to an ingest module. * - * @param module The ingest module to be decorated. + * @param module The ingest module to be wrapped. * @param displayName The display name for the module. */ PipelineModule(IngestModule module, String displayName) { @@ -303,15 +298,12 @@ abstract class IngestTaskPipeline { } /** - * Gets the the processing start time for the wrapped module, as set by + * Gets the the processing start time for the wrapped module. * * @return The start time, will be null if the module has not started * processing the data source yet. */ Date getProcessingStartTime() { - if (processingStartTime == null) { - throw new IllegalStateException("setProcessingStartTime() was not called"); //NON-NLS - } return new Date(processingStartTime.getTime()); } @@ -321,7 +313,7 @@ abstract class IngestTaskPipeline { } /** - * Processes an ingest task. + * Performs an ingest task. * * @param ingestJobPipeline The ingest job pipeline that owns the ingest * module pipeline this module belongs to.