Merge branch 'develop' into 7279-updateOSAccountTreeToRefresh

Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java

@@ -162,8 +162,7 @@ final class ArtifactsListPanel extends AbstractArtifactListPanel {
     @ThreadConfined(type = ThreadConfined.ThreadType.AWT)
     @Override
     void clearList() {
-        tableModel.setContents(new ArrayList<>());
+        addArtifacts(new ArrayList<>());
-        tableModel.fireTableDataChanged();
     }
 
     /**

Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java

@@ -80,9 +80,9 @@ final class DomainDetailsPanel extends JPanel {
                     if (selectedTabName == null || !selectedTabName.equals(newTabTitle)) {
                         selectedTabName = newTabTitle;
                         Component selectedComponent = jTabbedPane1.getSelectedComponent();
-                        if (selectedComponent instanceof DomainArtifactsTabPanel) {
+                        if (!StringUtils.isBlank(domain) && selectedComponent instanceof DomainArtifactsTabPanel) {
                             runDomainWorker((DomainArtifactsTabPanel) selectedComponent, true);
-                        } else if (selectedComponent instanceof MiniTimelinePanel) {
+                        } else if (!StringUtils.isBlank(domain) && selectedComponent instanceof MiniTimelinePanel) {
                             runMiniTimelineWorker((MiniTimelinePanel) selectedComponent, true);
                         }
                     }
@@ -170,13 +170,13 @@ final class DomainDetailsPanel extends JPanel {
     @Subscribe
     void handlePopulateDomainTabsEvent(DiscoveryEventUtils.PopulateDomainTabsEvent populateEvent) {
         SwingUtilities.invokeLater(() -> {
-            if (StringUtils.isBlank(populateEvent.getDomain())) {
+            domain = populateEvent.getDomain();
+            if (StringUtils.isBlank(domain)) {
                 resetTabsStatus();
                 //send fade out event
                 DiscoveryEventUtils.getDiscoveryEventBus().post(new DiscoveryEventUtils.DetailsVisibleEvent(false));
             } else {
                 resetTabsStatus();
-                domain = populateEvent.getDomain();
                 Component selectedComponent = jTabbedPane1.getSelectedComponent();
                 if (selectedComponent instanceof DomainArtifactsTabPanel) {
                     runDomainWorker((DomainArtifactsTabPanel) selectedComponent, false);

Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java

@@ -23,6 +23,11 @@ import org.sleuthkit.datamodel.Content;
 /**
  * An adapter that provides a no-op implementation of the startUp() method for
  * data source ingest modules.
+ *
+ * NOTE: As of Java 8, interfaces can have default methods.
+ * DataSourceIngestModule now provides default no-op versions of startUp() and
+ * shutDown(). This class is no longer needed and can be deprecated when
+ * convenient.
  */
 public abstract class DataSourceIngestModuleAdapter implements DataSourceIngestModule {
 

Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java

@@ -1,15 +1,15 @@
 /*
  * Autopsy Forensic Browser
- * 
+ *
- * Copyright 2011-2016 Basis Technology Corp.
+ * Copyright 2014-2021 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,184 +18,86 @@
  */
 package org.sleuthkit.autopsy.ingest;
 
-import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
+import java.util.Optional;
 import java.util.logging.Level;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.datamodel.Content;
 
 /**
- * This class manages a sequence of data source level ingest modules for an
+ * A pipeline of data source level ingest modules for performing data source
- *  ingestJobPipeline. It starts the modules, runs data sources through them, and
+ * level ingest tasks for an ingest job.
- * shuts them down when data source level ingest is complete.
- * <p>
- * This class is thread-safe.
  */
-final class DataSourceIngestPipeline {
+final class DataSourceIngestPipeline extends IngestTaskPipeline<DataSourceIngestTask> {
 
-    private static final IngestManager ingestManager = IngestManager.getInstance();
     private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName());
-    private final IngestJobPipeline ingestJobPipeline;
+    private static final IngestManager ingestManager = IngestManager.getInstance();
-    private final List<PipelineModule> modules = new ArrayList<>();
-    private volatile PipelineModule currentModule;
 
     /**
-     * Constructs an object that manages a sequence of data source level ingest
+     * Constructs a pipeline of data source level ingest modules for performing
-     * modules. It starts the modules, runs data sources through them, and shuts
+     * data source level ingest tasks for an ingest job.
-     * them down when data source level ingest is complete.
      *
-     * @param ingestJobPipeline  The ingestJobPipeline that owns this pipeline.
+     * @param ingestJobPipeline The ingest job pipeline that owns this pipeline.
-     * @param moduleTemplates Templates for the creating the ingest modules that
+     * @param moduleTemplates   The ingest module templates that define this
-     *                        make up this pipeline.
+     *                          pipeline.
      */
     DataSourceIngestPipeline(IngestJobPipeline ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
-        this.ingestJobPipeline = ingestJobPipeline;
+        super(ingestJobPipeline, moduleTemplates);
-        for (IngestModuleTemplate template : moduleTemplates) {
-            if (template.isDataSourceIngestModuleTemplate()) {
-                PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName());
-                modules.add(module);
-            }
-        }
     }
 
-    /**
+    @Override
-     * Indicates whether or not there are any ingest modules in this pipeline.
+    Optional<IngestTaskPipeline.PipelineModule<DataSourceIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
-     *
+        Optional<IngestTaskPipeline.PipelineModule<DataSourceIngestTask>> module = Optional.empty();
-     * @return True or false.
+        if (template.isDataSourceIngestModuleTemplate()) {
-     */
+            DataSourceIngestModule ingestModule = template.createDataSourceIngestModule();
-    boolean isEmpty() {
+            module = Optional.of(new DataSourcePipelineModule(ingestModule, template.getModuleName()));
-        return modules.isEmpty();
+        }
+        return module;
     }
 
-    /**
+    @Override
-     * Starts up the ingest modules in this pipeline.
+    void prepareTask(DataSourceIngestTask task) {
-     *
-     * @return A list of ingest module startup errors, possibly empty.
-     */
-    synchronized List<IngestModuleError> startUp() {
-        List<IngestModuleError> errors = new ArrayList<>();
-        for (PipelineModule module : modules) {
-            try {
-                module.startUp(new IngestJobContext(this.ingestJobPipeline));
-            } catch (Throwable ex) { // Catch-all exception firewall
-                errors.add(new IngestModuleError(module.getDisplayName(), ex));
-            }
-        }
-        return errors;
     }
 
-    /**
+    @Override
-     * Runs a data source through the ingest modules in sequential order.
+    void completeTask(DataSourceIngestTask task) {
-     *
-     * @param task A data source level ingest task containing a data source to
-     *             be processed.
-     *
-     * @return A list of processing errors, possible empty.
-     */
-    synchronized List<IngestModuleError> process(DataSourceIngestTask task) {
-        List<IngestModuleError> errors = new ArrayList<>();
-        if (!this.ingestJobPipeline.isCancelled()) {
-            Content dataSource = task.getDataSource();
-            for (PipelineModule module : modules) {
-                try {
-                    this.currentModule = module;
-                    String displayName = NbBundle.getMessage(this.getClass(),
-                            "IngestJob.progress.dataSourceIngest.displayName",
-                            module.getDisplayName(), dataSource.getName());
-                    this.ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(displayName);
-                    this.ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate();
-                    DataSourceIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName());
-                    logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) starting", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS
-                    module.process(dataSource, new DataSourceIngestModuleProgress(this.ingestJobPipeline));
-                    logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) finished", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS
-                } catch (Throwable ex) { // Catch-all exception firewall
-                    errors.add(new IngestModuleError(module.getDisplayName(), ex));
-                }
-                if (this.ingestJobPipeline.isCancelled()) {
-                    break;
-                } else if (this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) {
-                    this.ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(currentModule.getDisplayName());
-                }
-            }
-        }
-        this.currentModule = null;
         ingestManager.setIngestTaskProgressCompleted(task);
-        return errors;
     }
 
     /**
-     * Gets the currently running module.
+     * A wrapper that adds ingest infrastructure operations to a data source
-     *
+     * level ingest module.
-     * @return The module, possibly null if no module is currently running.
      */
-    PipelineModule getCurrentlyRunningModule() {
+    static final class DataSourcePipelineModule extends IngestTaskPipeline.PipelineModule<DataSourceIngestTask> {
-        return this.currentModule;
-    }
-
-    /**
-     * This class decorates a data source level ingest module with a display
-     * name and a processing start time.
-     */
-    static class PipelineModule implements DataSourceIngestModule {
 
         private final DataSourceIngestModule module;
-        private final String displayName;
-        private volatile Date processingStartTime;
 
         /**
-         * Constructs an object that decorates a data source level ingest module
+         * Constructs a wrapper that adds ingest infrastructure operations to a
-         * with a display name and a processing start time.
+         * data source level ingest module.
-         *
-         * @param module      The data source level ingest module to be
-         *                    decorated.
-         * @param displayName The display name.
          */
-        PipelineModule(DataSourceIngestModule module, String displayName) {
+        DataSourcePipelineModule(DataSourceIngestModule module, String displayName) {
+            super(module, displayName);
             this.module = module;
-            this.displayName = displayName;
-            this.processingStartTime = new Date();
-        }
-
-        /**
-         * Gets the class name of the decorated ingest module.
-         *
-         * @return The class name.
-         */
-        String getClassName() {
-            return this.module.getClass().getCanonicalName();
-        }
-
-        /**
-         * Gets the display of the decorated ingest module.
-         *
-         * @return The display name.
-         */
-        String getDisplayName() {
-            return this.displayName;
-        }
-
-        /**
-         * Gets the time the decorated ingest module started processing the data
-         * source.
-         *
-         * @return The start time, will be null if the module has not started
-         *         processing the data source yet.
-         */
-        Date getProcessingStartTime() {
-            return this.processingStartTime;
         }
 
         @Override
-        public void startUp(IngestJobContext context) throws IngestModuleException {
+        void performTask(IngestJobPipeline ingestJobPipeline, DataSourceIngestTask task) throws IngestModuleException {
-            this.module.startUp(context);
+            Content dataSource = task.getDataSource();
-        }
+            String progressBarDisplayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", getDisplayName(), dataSource.getName());
-
+            ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(progressBarDisplayName);
-        @Override
+            ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate();
-        public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
+            ingestManager.setIngestTaskProgress(task, getDisplayName());
-            this.processingStartTime = new Date();
+            logger.log(Level.INFO, "{0} analysis of {1} starting", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS
-            return this.module.process(dataSource, statusHelper);
+            ProcessResult result = module.process(dataSource, new DataSourceIngestModuleProgress(ingestJobPipeline));
+            logger.log(Level.INFO, "{0} analysis of {1} finished", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS            
+            if (!ingestJobPipeline.isCancelled() && ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) {
+                ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(getDisplayName());
+            }
+            if (result == ProcessResult.ERROR) {
+                throw new IngestModuleException(String.format("%s experienced an error analyzing %s (data source objId = %d)", getDisplayName(), dataSource.getName(), dataSource.getId())); //NON-NLS
+            }            
         }
 
     }

Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2014 Basis Technology Corp.
+ * Copyright 2014-2021 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -36,13 +36,4 @@ public interface FileIngestModule extends IngestModule {
      */
     ProcessResult process(AbstractFile file);
 
-    /**
-     * Invoked by Autopsy when an ingest job is completed (either because the
-     * data has been analyzed or because the job was canceled - check
-     * IngestJobContext.fileIngestIsCancelled()), before the ingest module
-     * instance is discarded. The module should respond by doing things like
-     * releasing private resources, submitting final results, and posting a
-     * final ingest message.
-     */
-    void shutDown();
 }

Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java

@@ -23,6 +23,10 @@ import org.sleuthkit.datamodel.AbstractFile;
 /**
  * An adapter that provides no-op implementations of the startUp() and
  * shutDown() methods for file ingest modules.
+ *
+ * NOTE: As of Java 8, interfaces can have default methods. FileIngestModule now
+ * provides default no-op versions of startUp() and shutDown(). This class is no
+ * longer needed and can be deprecated when convenient.
  */
 public abstract class FileIngestModuleAdapter implements FileIngestModule {
 

Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java

@@ -1,15 +1,15 @@
 /*
  * Autopsy Forensic Browser
- * 
+ *
- * Copyright 2014-2015 Basis Technology Corp.
+ * Copyright 2014-2021 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,223 +18,109 @@
  */
 package org.sleuthkit.autopsy.ingest;
 
-import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
-import java.util.logging.Level;
+import java.util.Optional;
-
-import org.openide.util.NbBundle;
-import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
- * This class manages a sequence of file level ingest modules for an
+ * A pipeline of file ingest modules for performing file ingest tasks for an
- * ingest job pipeline. It starts the modules, runs files through them, and shuts them
+ * ingest job.
- * down when file level ingest is complete.
- * <p>
- * This class is thread-safe.
  */
-final class FileIngestPipeline {
+final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
 
     private static final IngestManager ingestManager = IngestManager.getInstance();
     private final IngestJobPipeline ingestJobPipeline;
-    private final List<PipelineModule> modules = new ArrayList<>();
-    private Date startTime;
-    private volatile boolean running;
 
     /**
-     * Constructs an object that manages a sequence of file level ingest
+     * Constructs a pipeline of file ingest modules for performing file ingest
-     * modules. It starts the modules, runs files through them, and shuts them
+     * tasks for an ingest job.
-     * down when file level ingest is complete.
      *
-     * @param ingestJobPipeline  The ingestJobPipeline that owns the pipeline.
+     * @param ingestJobPipeline The ingest job pipeline that owns this pipeline.
-     * @param moduleTemplates The ingest module templates that define the
+     * @param moduleTemplates   The ingest module templates that define this
-     *                        pipeline.
+     *                          pipeline.
      */
     FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
+        super(ingestJobPipeline, moduleTemplates);
         this.ingestJobPipeline = ingestJobPipeline;
-        for (IngestModuleTemplate template : moduleTemplates) {
+    }
-            if (template.isFileIngestModuleTemplate()) {
+
-                PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName());
+    @Override
-                modules.add(module);
+    Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
-            }
+        Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> module = Optional.empty();
+        if (template.isFileIngestModuleTemplate()) {
+            FileIngestModule ingestModule = template.createFileIngestModule();
+            module = Optional.of(new FileIngestPipelineModule(ingestModule, template.getModuleName()));
         }
+        return module;
     }
 
-    /**
+    @Override
-     * Queries whether or not there are any ingest modules in this pipeline.
+    void prepareTask(FileIngestTask task) throws IngestTaskPipelineException {
-     *
-     * @return True or false.
-     */
-    boolean isEmpty() {
-        return this.modules.isEmpty();
     }
 
-    /**
+    @Override
-     * Queries whether or not this pipeline is running.
+    void completeTask(FileIngestTask task) throws IngestTaskPipelineException {
-     *
+        AbstractFile file = null;
-     * @return True or false.
+        try {
-     */
+            file = task.getFile();
-    boolean isRunning() {
+        } catch (TskCoreException ex) {
-        return this.running;
+            throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
-    }
-
-    /**
-     * Returns the start up time of this pipeline.
-     *
-     * @return The file processing start time, may be null if this pipeline has
-     *         not been started yet.
-     */
-    Date getStartTime() {
-        return this.startTime;
-    }
-
-    /**
-     * Starts up all of the ingest modules in the pipeline.
-     *
-     * @return List of start up errors, possibly empty.
-     */
-    synchronized List<IngestModuleError> startUp() {
-        this.startTime = new Date();
-        this.running = true;
-        List<IngestModuleError> errors = new ArrayList<>();
-        for (PipelineModule module : this.modules) {
-            try {
-                module.startUp(new IngestJobContext(this.ingestJobPipeline));
-            } catch (Throwable ex) { // Catch-all exception firewall
-                errors.add(new IngestModuleError(module.getDisplayName(), ex));
-            }
         }
-        return errors;
+        try {
-    }
+            if (!ingestJobPipeline.isCancelled()) {
-
+                /*
-    /**
+                 * Save any updates from the ingest modules to the case
-     * Runs a file through the ingest modules in sequential order.
+                 * database.
-     *
+                 */
-     * @param task A file level ingest task containing a file to be processed.
+                file.save();
-     *
-     * @return A list of processing errors, possible empty.
-     */
-    synchronized List<IngestModuleError> process(FileIngestTask task) {
-        List<IngestModuleError> errors = new ArrayList<>();
-        if (!this.ingestJobPipeline.isCancelled()) {
-            AbstractFile file;
-            try {
-                file = task.getFile();
-            } catch (TskCoreException ex) {
-                // In practice, this task would never have been enqueued since the file
-                // lookup would have failed there.
-                errors.add(new IngestModuleError("File Ingest Pipeline", ex)); // NON-NLS
-                FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task);
-                return errors;
             }
-            for (PipelineModule module : this.modules) {
+        } catch (TskCoreException ex) {
-                try {
+            throw new IngestTaskPipelineException(String.format("Failed to save updated data for file (file objId = %d)", task.getFileId()), ex); //NON-NLS
-                    FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName());
+        } finally {
-                    this.ingestJobPipeline.setCurrentFileIngestModule(module.getDisplayName(), task.getFile().getName());
+            if (!ingestJobPipeline.isCancelled()) {
-                    module.process(file);
-                } catch (Throwable ex) { // Catch-all exception firewall
-                    errors.add(new IngestModuleError(module.getDisplayName(), ex));
-                }
-                if (this.ingestJobPipeline.isCancelled()) {
-                    break;
-                }
-            }
-            
-            if (!this.ingestJobPipeline.isCancelled()) {
-                // Save any properties that have not already been saved to the database
-                try{
-                    file.save();
-                } catch (TskCoreException ex){
-                    Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data for file " + file.getId(), ex); //NON-NLS
-                }
                 IngestManager.getInstance().fireFileIngestDone(file);
             }
             file.close();
+            ingestManager.setIngestTaskProgressCompleted(task);
         }
-        FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task);
-        return errors;
     }
 
     /**
-     * Shuts down all of the modules in the pipeline.
+     * A wrapper that adds ingest infrastructure operations to a file ingest
-     *
+     * module.
-     * @return A list of shut down errors, possibly empty.
      */
-    synchronized List<IngestModuleError> shutDown() {
+    static final class FileIngestPipelineModule extends IngestTaskPipeline.PipelineModule<FileIngestTask> {
-        List<IngestModuleError> errors = new ArrayList<>();
-        if (this.running == true) { // Don't shut down pipelines that never started
-            for (PipelineModule module : this.modules) {
-                try {
-                    module.shutDown();
-                } catch (Throwable ex) { // Catch-all exception firewall
-                    errors.add(new IngestModuleError(module.getDisplayName(), ex));
-                    String msg = ex.getMessage();
-                    // Jython run-time errors don't seem to have a message, but have details in toString.
-                    if (msg == null) {
-                        msg = ex.toString();
-                    }
-                    MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg);
-                }
-            }
-        }
-        this.running = false;
-        return errors;
-    }
-
-    /**
-     * This class decorates a file level ingest module with a display name.
-     */
-    private static final class PipelineModule implements FileIngestModule {
 
         private final FileIngestModule module;
-        private final String displayName;
 
         /**
-         * Constructs an object that decorates a file level ingest module with a
+         * Constructs a wrapper that adds ingest infrastructure operations to a
-         * display name.
+         * file ingest module.
          *
-         * @param module      The file level ingest module to be decorated.
+         *
-         * @param displayName The display name.
+         * @param module      The module.
+         * @param displayName The display name of the module.
          */
-        PipelineModule(FileIngestModule module, String displayName) {
+        FileIngestPipelineModule(FileIngestModule module, String displayName) {
+            super(module, displayName);
             this.module = module;
-            this.displayName = displayName;
-        }
-
-        /**
-         * Gets the class name of the decorated ingest module.
-         *
-         * @return The class name.
-         */
-        String getClassName() {
-            return module.getClass().getCanonicalName();
-        }
-
-        /**
-         * Gets the display name of the decorated ingest module.
-         *
-         * @return The display name.
-         */
-        String getDisplayName() {
-            return displayName;
         }
 
         @Override
-        public void startUp(IngestJobContext context) throws IngestModuleException {
+        void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException {
-            module.startUp(context);
+            AbstractFile file = null;
-        }
+            try {
-
+                file = task.getFile();
-        @Override
+            } catch (TskCoreException ex) {
-        public IngestModule.ProcessResult process(AbstractFile file) {
+                throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
-            return module.process(file);
+            }
-        }
+            ingestManager.setIngestTaskProgress(task, getDisplayName());
-
+            ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName());
-        @Override
+            ProcessResult result = module.process(file);
-        public void shutDown() {
+            if (result == ProcessResult.ERROR) {
-            module.shutDown();
+                throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS
+            }
         }
 
     }

Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java

@@ -416,7 +416,7 @@ public final class IngestJob {
                 Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot);
                 dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot));
                 if (null == dataSourceModule) {
-                    DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule();
+                    DataSourceIngestPipeline.DataSourcePipelineModule module = snapshot.getDataSourceLevelIngestModule();
                     if (null != module) {
                         dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module);
                     }
@@ -500,7 +500,7 @@ public final class IngestJob {
     public static class DataSourceIngestModuleHandle {
 
         private final IngestJobPipeline ingestJobPipeline;
-        private final DataSourceIngestPipeline.PipelineModule module;
+        private final DataSourceIngestPipeline.DataSourcePipelineModule module;
         private final boolean cancelled;
 
         /**
@@ -511,7 +511,7 @@ public final class IngestJob {
          * @param ingestJobPipeline    The ingestJobPipeline that owns the data source level ingest module.
          * @param module The data source level ingest module.
          */
-        private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.PipelineModule module) {
+        private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) {
             this.ingestJobPipeline = ingestJobPipeline;
             this.module = module;
             this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled();

Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2014-2019 Basis Technology Corp.
+ * Copyright 2014-2021 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -553,7 +553,7 @@ final class IngestJobPipeline {
 
         /*
          * If the data-source-level ingest pipelines were successfully started,
-         * start the Start the file-level ingest pipelines (one per file ingest
+         * start the file-level ingest pipelines (one per pipeline file ingest
          * thread).
          */
         if (errors.isEmpty()) {
@@ -940,7 +940,7 @@ final class IngestJobPipeline {
             synchronized (this.dataSourceIngestPipelineLock) {
                 if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
                     List<IngestModuleError> errors = new ArrayList<>();
-                    errors.addAll(this.currentDataSourceIngestPipeline.process(task));
+                    errors.addAll(this.currentDataSourceIngestPipeline.performTask(task));
                     if (!errors.isEmpty()) {
                         logIngestModuleErrors(errors);
                     }
@@ -1014,7 +1014,7 @@ final class IngestJobPipeline {
                      * Run the file through the pipeline.
                      */
                     List<IngestModuleError> errors = new ArrayList<>();
-                    errors.addAll(pipeline.process(task));
+                    errors.addAll(pipeline.performTask(task));
                     if (!errors.isEmpty()) {
                         logIngestModuleErrors(errors, file);
                     }
@@ -1232,9 +1232,9 @@ final class IngestJobPipeline {
      *
      * @return The currently running module, may be null.
      */
-    DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
+    DataSourceIngestPipeline.DataSourcePipelineModule getCurrentDataSourceIngestModule() {
-        if (null != this.currentDataSourceIngestPipeline) {
+        if (null != currentDataSourceIngestPipeline) {
-            return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
+            return (DataSourceIngestPipeline.DataSourcePipelineModule) currentDataSourceIngestPipeline.getCurrentlyRunningModule();
         } else {
             return null;
         }
@@ -1274,7 +1274,7 @@ final class IngestJobPipeline {
                 }
             }
         }
-        
+
         // If a data source had no tasks in progress it may now be complete.
         checkForStageCompleted();
     }
@@ -1353,18 +1353,18 @@ final class IngestJobPipeline {
             logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis", error.getModuleDisplayName()), error.getThrowable()); //NON-NLS
         }
     }
-    
+
     /**
      * Write ingest module errors to the log.
      *
      * @param errors The errors.
-     * @param file AbstractFile that caused the errors.
+     * @param file   AbstractFile that caused the errors.
      */
     private void logIngestModuleErrors(List<IngestModuleError> errors, AbstractFile file) {
         for (IngestModuleError error : errors) {
             logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s, object ID %d", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS
         }
-    }    
+    }
 
     /**
      * Gets a snapshot of this jobs state and performance.

Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2012-2019 Basis Technology Corp.
+ * Copyright 2012-2021 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2014 Basis Technology Corp.
+ * Copyright 2014-2021 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,9 +25,8 @@ package org.sleuthkit.autopsy.ingest;
  * ingest job it performs (one for each thread that it is using).
  *
  * Autopsy will call startUp() before any data is processed, will pass any data
- * to be analyzed into the process() method (FileIngestModule.process() or
+ * to be analyzed into the process() method, and call shutDown() after either
- * DataSourceIngestModule.process()), and call shutDown() after either all data
+ * all data is analyzed or the user has canceled the job.
- * is analyzed or the user has canceled the job.
  *
  * Autopsy may use multiple threads to complete an ingest job, but it is
  * guaranteed that a module instance will always be called from a single thread.
@@ -47,25 +46,53 @@ package org.sleuthkit.autopsy.ingest;
 public interface IngestModule {
 
     /**
-     * A return code for derived class process() methods.
+     * Invoked by Autopsy to allow an ingest module instance to set up any
+     * internal data structures and acquire any private resources it will need
+     * during an ingest job. If the module depends on loading any resources, it
+     * should do so in this method so that it can throw an exception in the case
+     * of an error and alert the user. Exceptions that are thrown from startUp()
+     * are logged and stop processing of the data source.
+     *
+     * IMPORTANT: If the module instances must share resources, the modules are
+     * responsible for synchronizing access to the shared resources and doing
+     * reference counting as required to release those resources correctly.
+     * Also, more than one ingest job may be in progress at any given time. This
+     * must also be taken into consideration when sharing resources between
+     * module instances. See IngestModuleReferenceCounter.
+     *
+     * @param context Provides data and services specific to the ingest job and
+     *                the ingest pipeline of which the module is a part.
+     *
+     * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
      */
-    public enum ProcessResult {
+    default void startUp(IngestJobContext context) throws IngestModuleException {
-
+    }
-        OK,
-        ERROR
-    };
 
     /**
-     * A custom exception for the use of ingest modules.
+     * Invoked by Autopsy when an ingest job is completed (either because the
+     * data has been analyzed or because the job was cancelled), before the
+     * ingest module instance is discarded. The module should respond by doing
+     * things like releasing private resources, submitting final results, and
+     * posting a final ingest message.
+     * 
+     * IMPORTANT: If the module instances must share resources, the modules are
+     * responsible for synchronizing access to the shared resources and doing
+     * reference counting as required to release those resources correctly.
+     * Also, more than one ingest job may be in progress at any given time. This
+     * must also be taken into consideration when sharing resources between
+     * module instances. See IngestModuleReferenceCounter.
+     *
+     */
+    default void shutDown() {
+    }
+
+    /**
+     * An exception for the use of ingest modules.
      */
     public class IngestModuleException extends Exception {
 
         private static final long serialVersionUID = 1L;
 
-        @Deprecated
-        public IngestModuleException() {
-        }
-
         public IngestModuleException(String message) {
             super(message);
         }
@@ -73,26 +100,21 @@ public interface IngestModule {
         public IngestModuleException(String message, Throwable cause) {
             super(message, cause);
         }
+
+        @Deprecated
+        public IngestModuleException() {
+        }
+
     }
 
     /**
-     * Invoked by Autopsy to allow an ingest module instance to set up any
+     * A return code for subclass process() methods.
-     * internal data structures and acquire any private resources it will need
-     * during an ingest job. If the module depends on loading any resources, it
-     * should do so in this method so that it can throw an exception in the case
-     * of an error and alert the user. Exceptions that are thrown from process()
-     * and shutDown() are logged, but do not stop processing of the data source.
-     *
-     * @param context Provides data and services specific to the ingest job and
-     *                the ingest pipeline of which the module is a part.
-     *
-     * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
      */
-    void startUp(IngestJobContext context) throws IngestModuleException;
+    public enum ProcessResult {
+
+        OK,
+        ERROR
+
+    };
 
-    /**
-     * TODO: The next time an API change is legal, add a cancel() method and
-     * remove the "ingest job is canceled" queries from the IngestJobContext
-     * class.
-     */
 }

Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java

@@ -0,0 +1,345 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2021 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.ingest;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Optional;
+import org.openide.util.NbBundle;
+import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
+
+/**
+ * An abstract superclass for pipelines of ingest modules for a given ingest
+ * task type. Some examples of ingest task types: data source level ingest
+ * tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses need to
+ * implement a specialization of the inner PipelineModule abstract superclass
+ * for the type of ingest modules that make up the pipeline.
+ *
+ * @param <T> The ingest task type.
+ */
+abstract class IngestTaskPipeline<T extends IngestTask> {
+
+    private static final IngestManager ingestManager = IngestManager.getInstance();
+    private final IngestJobPipeline ingestJobPipeline;
+    private final List<IngestModuleTemplate> moduleTemplates;
+    private final List<PipelineModule<T>> modules;
+    private volatile Date startTime;
+    private volatile boolean running;
+    private volatile PipelineModule<T> currentModule;
+
+    /**
+     * Constructs an instance of an abstract superclass for pipelines of ingest
+     * modules for a given ingest task type. Some examples of ingest task types:
+     * data source level ingest tasks, file ingest tasks, data artifact ingest
+     * tasks, etc. Subclasses need to implement a specialization of the inner
+     * PipelineModule abstract superclass for the type of ingest modules that
+     * make up the pipeline.
+     *
+     * @param ingestJobPipeline The ingest job pipeline that owns this pipeline.
+     * @param moduleTemplates   The ingest module templates that define this
+     *                          pipeline.
+     */
+    IngestTaskPipeline(IngestJobPipeline ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
+        this.ingestJobPipeline = ingestJobPipeline;
+        this.moduleTemplates = moduleTemplates;
+        modules = new ArrayList<>();
+    }
+
+    /**
+     * Indicates whether or not there are any ingest modules in this pipeline.
+     *
+     * @return True or false.
+     */
+    boolean isEmpty() {
+        return modules.isEmpty();
+    }
+
+    /**
+     * Queries whether or not this pipeline is running, i.e., started and not
+     * shut down.
+     *
+     * @return True or false.
+     */
+    boolean isRunning() {
+        return running;
+    }
+
+    /**
+     * Starts up the ingest modules in this pipeline.
+     *
+     * @return A list of ingest module startup errors, possibly empty.
+     */
+    List<IngestModuleError> startUp() {
+        createIngestModules(moduleTemplates);
+        return startUpIngestModules();
+    }
+
+    /**
+     * Creates the ingest modules for this pipeline.
+     *
+     * @param moduleTemplates The ingest module templates avaialble to this
+     *                        pipeline.
+     */
+    private void createIngestModules(List<IngestModuleTemplate> moduleTemplates) {
+        for (IngestModuleTemplate template : moduleTemplates) {
+            Optional<PipelineModule<T>> module = acceptModuleTemplate(template);
+            if (module.isPresent()) {
+                modules.add(module.get());
+            }
+        }
+    }
+
+    /**
+     * Determines if the type of ingest module that can be created from a given
+     * ingest module template should be added to this pipeline. If so, the
+     * ingest module is created and returned.
+     *
+     * @param ingestModuleTemplate The ingest module template to be used or
+     *                             ignored, as appropriate to the pipeline type.
+     *
+     * @return An Optional that is either empty or contains a newly created and
+     *         wrapped ingest module.
+     */
+    abstract Optional<PipelineModule<T>> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate);
+
+    /**
+     * Starts up the ingest modules in the pipeline.
+     *
+     * @return A list of ingest module startup errors, possibly empty.
+     */
+    private List<IngestModuleError> startUpIngestModules() {
+        startTime = new Date();
+        running = true;
+        List<IngestModuleError> errors = new ArrayList<>();
+        for (PipelineModule<T> module : modules) {
+            try {
+                module.startUp(new IngestJobContext(ingestJobPipeline));
+            } catch (Throwable ex) { // Catch-all exception firewall
+                errors.add(new IngestModuleError(module.getDisplayName(), ex));
+            }
+        }
+        return errors;
+    }
+
+    /**
+     * Returns the start up time of this pipeline.
+     *
+     * @return The file processing start time, may be null if this pipeline has
+     *         not been started yet.
+     */
+    Date getStartTime() {
+        if (startTime == null) {
+            throw new IllegalStateException("startUp() was not called"); //NON-NLS
+        }
+        return new Date(startTime.getTime());
+    }
+
+    /**
+     * Does any preparation required before performing a task.
+     *
+     * @param task The task.
+     *
+     * @throws IngestTaskPipelineException Thrown if there is an error preparing
+     *                                     to perform the task.
+     */
+    abstract void prepareTask(T task) throws IngestTaskPipelineException;
+
+    /**
+     * Performs an ingest task using the ingest modules in this pipeline.
+     *
+     * @param task The task.
+     *
+     * @return A list of ingest module task processing errors, possibly empty.
+     */
+    List<IngestModuleError> performTask(T task) {
+        List<IngestModuleError> errors = new ArrayList<>();
+        if (!this.ingestJobPipeline.isCancelled()) {
+            try {
+                prepareTask(task);
+            } catch (IngestTaskPipelineException ex) {
+                errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS
+                return errors;
+            }
+            for (PipelineModule<T> module : modules) {
+                try {
+                    currentModule = module;
+                    currentModule.setProcessingStartTime();
+                    module.performTask(ingestJobPipeline, task);
+                } catch (Throwable ex) { // Catch-all exception firewall
+                    errors.add(new IngestModuleError(module.getDisplayName(), ex));
+                }
+                if (ingestJobPipeline.isCancelled()) {
+                    break;
+                }
+            }
+        }
+        try {
+            completeTask(task);
+        } catch (IngestTaskPipelineException ex) {
+            errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS
+        }
+        currentModule = null;
+        return errors;
+    }
+
+    /**
+     * Gets the currently running module.
+     *
+     * @return The module, possibly null if no module is currently running.
+     */
+    PipelineModule<T> getCurrentlyRunningModule() {
+        return currentModule;
+    }
+
+    /**
+     * Does any clean up required after performing a task.
+     *
+     * @param task The task.
+     *
+     * @throws IngestTaskPipelineException Thrown if there is an error cleaning
+     *                                     up after performing the task.
+     */
+    abstract void completeTask(T task) throws IngestTaskPipelineException;
+
+    /**
+     * Shuts down all of the modules in the pipeline.
+     *
+     * @return A list of shut down errors, possibly empty.
+     */
+    List<IngestModuleError> shutDown() {
+        List<IngestModuleError> errors = new ArrayList<>();
+        if (running == true) {
+            for (PipelineModule<T> module : modules) {
+                try {
+                    module.shutDown();
+                } catch (Throwable ex) { // Catch-all exception firewall
+                    errors.add(new IngestModuleError(module.getDisplayName(), ex));
+                    String msg = ex.getMessage();
+                    if (msg == null) {
+                        /*
+                         * Jython run-time errors don't seem to have a message,
+                         * but have details in the string returned by
+                         * toString().
+                         */
+                        msg = ex.toString();
+                    }
+                    MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg);
+                }
+            }
+        }
+        running = false;
+        return errors;
+    }
+
+    /**
+     * An abstract superclass for a wrapper that adds ingest infrastructure
+     * operations to an ingest module.
+     */
+    static abstract class PipelineModule<T extends IngestTask> implements IngestModule {
+
+        private final IngestModule module;
+        private final String displayName;
+        private volatile Date processingStartTime;
+
+        /**
+         * Constructs an instance of an abstract superclass for a wrapper that
+         * adds ingest infrastructure operations to an ingest module.
+         *
+         * @param module      The ingest module to be wrapped.
+         * @param displayName The display name for the module.
+         */
+        PipelineModule(IngestModule module, String displayName) {
+            this.module = module;
+            this.displayName = displayName;
+            this.processingStartTime = new Date();
+        }
+
+        /**
+         * Gets the class name of the wrapped ingest module.
+         *
+         * @return The class name.
+         */
+        String getClassName() {
+            return module.getClass().getCanonicalName();
+        }
+
+        /**
+         * Gets the display name of the wrapped ingest module.
+         *
+         * @return The display name.
+         */
+        String getDisplayName() {
+            return displayName;
+        }
+
+        /**
+         * Sets the processing start time for the wrapped module to the system
+         * time when this method is called.
+         */
+        void setProcessingStartTime() {
+            processingStartTime = new Date();
+        }
+
+        /**
+         * Gets the the processing start time for the wrapped module.
+         *
+         * @return The start time, will be null if the module has not started
+         *         processing the data source yet.
+         */
+        Date getProcessingStartTime() {
+            return new Date(processingStartTime.getTime());
+        }
+
+        @Override
+        public void startUp(IngestJobContext context) throws IngestModuleException {
+            module.startUp(context);
+        }
+
+        /**
+         * Performs an ingest task.
+         *
+         * @param ingestJobPipeline The ingest job pipeline that owns the ingest
+         *                          module pipeline this module belongs to.
+         * @param task              The task to process.
+         *
+         * @throws IngestModuleException Excepton thrown if there is an error
+         *                               performing the task.
+         */
+        abstract void performTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException;
+
+    }
+
+    /**
+     * An exception for the use of ingest task pipelines.
+     */
+    public static class IngestTaskPipelineException extends Exception {
+
+        private static final long serialVersionUID = 1L;
+
+        public IngestTaskPipelineException(String message) {
+            super(message);
+        }
+
+        public IngestTaskPipelineException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+}

Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java

@@ -34,7 +34,7 @@ public final class Snapshot implements Serializable {
     private final long jobId;
     private final long jobStartTime;
     private final long snapShotTime;
-    transient private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
+    transient private final DataSourceIngestPipeline.DataSourcePipelineModule dataSourceLevelIngestModule;
     private final boolean fileIngestRunning;
     private final Date fileIngestStartTime;
     private final long processedFiles;
@@ -48,7 +48,7 @@ public final class Snapshot implements Serializable {
      * Constructs an object to store basic diagnostic statistics for a data
      * source ingest job.
      */
-    Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.PipelineModule dataSourceIngestModule,
+    Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.DataSourcePipelineModule dataSourceIngestModule,
             boolean fileIngestRunning, Date fileIngestStartTime,
             boolean jobCancelled, IngestJob.CancellationReason cancellationReason, List<String> cancelledModules,
             long processedFiles, long estimatedFilesToProcess,
@@ -110,7 +110,7 @@ public final class Snapshot implements Serializable {
         return jobStartTime;
     }
 
-    DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
+    DataSourceIngestPipeline.DataSourcePipelineModule getDataSourceLevelIngestModule() {
         return this.dataSourceLevelIngestModule;
     }