Merge pull request #7554 from eugene7646/merge_develop

Merge develop

Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java

@@ -393,84 +393,68 @@ public final class ContentUtils {
             cntnt.accept(new ExtractFscContentVisitor<>(dest, progress, worker, true));
         }
         
-        @Override
+        /**
-        public Void visit(File file) {
+         * Base method writing a file to disk.
+         *
+         * @param file     The TSK content file.
+         * @param dest     The disk location where the content will be written.
+         * @param progress progress bar handle to update, if available. null
+         *                 otherwise
+         * @param worker   the swing worker background thread the process runs
+         *                 within, or null, if in the main thread, used to
+         *                 handle task cancellation
+         * @param source   true if source file
+         *
+         * @throws IOException
+         */
+        protected void writeFile(Content file, java.io.File dest, ProgressHandle progress, SwingWorker<T, V> worker, boolean source) throws IOException {
+            ContentUtils.writeToFile(file, dest, progress, worker, source);
+        }
+
+        /**
+         * Visits a TSK content file and writes that file to disk.
+         * @param file The file to be written.
+         * @param fileType The file type (i.e. "derived file") for error logging.
+         * @return null.
+         */
+        protected Void visitFile(Content file, String fileType) {
             try {
-                ContentUtils.writeToFile(file, dest, progress, worker, source);
+                writeFile(file, dest, progress, worker, source);
             } catch (ReadContentInputStreamException ex) {
                 logger.log(Level.WARNING,
                         String.format("Error reading file '%s' (id=%d).",
                                 file.getName(), file.getId()), ex); //NON-NLS
             } catch (IOException ex) {
                 logger.log(Level.SEVERE,
-                        String.format("Error extracting file '%s' (id=%d) to '%s'.",
+                        String.format("Error extracting %s '%s' (id=%d) to '%s'.",
-                                file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
+                                fileType, file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
             }
             return null;
         }
         
+        @Override
+        public Void visit(File file) {
+            return visitFile(file, "file");
+        }
+
         @Override
         public Void visit(LayoutFile file) {
-            try {
+            return visitFile(file, "unallocated content file");
-                ContentUtils.writeToFile(file, dest, progress, worker, source);
-            } catch (ReadContentInputStreamException ex) {
-                logger.log(Level.WARNING,
-                        String.format("Error reading file '%s' (id=%d).",
-                                file.getName(), file.getId()), ex); //NON-NLS
-            } catch (IOException ex) {
-                logger.log(Level.SEVERE,
-                        String.format("Error extracting unallocated content file '%s' (id=%d) to '%s'.",
-                                file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
-            }
-            return null;
         }
 
         @Override
         public Void visit(DerivedFile file) {
-            try {
+            return visitFile(file, "derived file");
-                ContentUtils.writeToFile(file, dest, progress, worker, source);
-            } catch (ReadContentInputStreamException ex) {
-                logger.log(Level.WARNING,
-                        String.format("Error reading file '%s' (id=%d).",
-                                file.getName(), file.getId()), ex); //NON-NLS
-            } catch (IOException ex) {
-                logger.log(Level.SEVERE,
-                        String.format("Error extracting derived file '%s' (id=%d) to '%s'.",
-                                file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
-            }
-            return null;
         }
 
         @Override
         public Void visit(LocalFile file) {
-            try {
+            return visitFile(file, "local file");
-                ContentUtils.writeToFile(file, dest, progress, worker, source);
-            } catch (ReadContentInputStreamException ex) {
-                logger.log(Level.WARNING,
-                        String.format("Error reading file '%s' (id=%d).",
-                                file.getName(), file.getId()), ex); //NON-NLS
-            } catch (IOException ex) {
-                logger.log(Level.SEVERE,
-                        String.format("Error extracting local file '%s' (id=%d) to '%s'.",
-                                file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
-            }
-            return null;
         }
 
         @Override
         public Void visit(SlackFile file) {
-            try {
+            return visitFile(file, "slack file");
-                ContentUtils.writeToFile(file, dest, progress, worker, source);
-            } catch (ReadContentInputStreamException ex) {
-                logger.log(Level.WARNING,
-                        String.format("Error reading file '%s' (id=%d).",
-                                file.getName(), file.getId()), ex); //NON-NLS
-            } catch (IOException ex) {
-                logger.log(Level.SEVERE,
-                        String.format("Error extracting slack file '%s' (id=%d) to '%s'.",
-                                file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
-            }
-            return null;
         }
 
         @Override
@@ -494,6 +478,20 @@ public final class ContentUtils {
             return new java.io.File(path);
         }
         
+        /**
+         * Returns a visitor to visit any child content.
+         * @param childFile     The disk location where the content will be written.
+         * @param progress progress bar handle to update, if available. null
+         *                 otherwise
+         * @param worker   the swing worker background thread the process runs
+         *                 within, or null, if in the main thread, used to
+         *                 handle task cancellation
+         * @return 
+         */
+        protected ExtractFscContentVisitor<T, V> getChildVisitor(java.io.File childFile, ProgressHandle progress, SwingWorker<T, V> worker) {
+            return new ExtractFscContentVisitor<>(childFile, progress, worker, false);
+        }
+
         public Void visitDir(AbstractFile dir) {
 
             // don't extract . and .. directories
@@ -509,8 +507,7 @@ public final class ContentUtils {
                 for (Content child : dir.getChildren()) {
                     if (child instanceof AbstractFile) { //ensure the directory's artifact children are ignored
                         java.io.File childFile = getFsContentDest(child);
-                        ExtractFscContentVisitor<T, V> childVisitor
+                        ExtractFscContentVisitor<T, V> childVisitor = getChildVisitor(childFile, progress, worker);
-                                = new ExtractFscContentVisitor<>(childFile, progress, worker, false);
                         // If this is the source directory of an extract it
                         // will have a progress and worker, and will keep track
                         // of the progress bar's progress

Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java

@@ -44,6 +44,8 @@ public final class ExtractAction extends AbstractAction {
         return instance;
     }
     
+    private final ExtractActionHelper extractor = new ExtractActionHelper();
+
     /**
      * Private constructor for the action.
      */
@@ -61,7 +63,6 @@ public final class ExtractAction extends AbstractAction {
     public void actionPerformed(ActionEvent e) {
         Lookup lookup = Utilities.actionsGlobalContext();
         Collection<? extends AbstractFile> selectedFiles =lookup.lookupAll(AbstractFile.class);
-        ExtractActionHelper extractor = new ExtractActionHelper();
         extractor.extract(e, selectedFiles);
 
     }

Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java

@@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.directorytree.actionhelpers;
 import java.awt.Component;
 import java.awt.event.ActionEvent;
 import java.io.File;
+import java.io.IOException;
+import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -42,8 +44,10 @@ import org.sleuthkit.autopsy.coreutils.FileUtil;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
 import org.sleuthkit.autopsy.datamodel.ContentUtils;
+import org.sleuthkit.autopsy.datamodel.ContentUtils.ExtractFscContentVisitor;
 import org.sleuthkit.autopsy.guiutils.JFileChooserFactory;
 import org.sleuthkit.datamodel.AbstractFile;
+import org.sleuthkit.datamodel.Content;
 
 /**
  * Helper class for methods needed by actions which extract files.
@@ -68,12 +72,7 @@ public class ExtractActionHelper {
         if (selectedFiles.size() > 1) {
             extractFiles(event, selectedFiles);
         } else if (selectedFiles.size() == 1) {
-            AbstractFile source = selectedFiles.iterator().next();
+            extractFile(event, selectedFiles.iterator().next());
-            if (source.isDir()) {
-                extractFiles(event, selectedFiles);
-            } else {
-                extractFile(event, selectedFiles.iterator().next());
-            }
         }
     }
 
@@ -83,7 +82,11 @@ public class ExtractActionHelper {
      * @param event
      * @param selectedFile Selected file
      */
-    @NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available."})
+    @NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available.",
+        "ExtractActionHelper.extractOverwrite.title=Export to csv file",
+        "# {0} - fileName",
+        "ExtractActionHelper.extractOverwrite.msg=A file already exists at {0}.  Do you want to overwrite the existing file?"
+    })
     private void extractFile(ActionEvent event, AbstractFile selectedFile) {
         Case openCase;
         try {
@@ -269,6 +272,68 @@ public class ExtractActionHelper {
         }
     }
 
+    /**
+     * A file content extraction visitor that handles for the UI designed to
+     * handle file name conflicts by appending the object id to the file name.
+     */
+    private static class UIExtractionVisitor<T, V> extends ExtractFscContentVisitor<T, V> {
+
+        /**
+         * @param file     The TSK content file.
+         * @param dest     The disk location where the content will be written.
+         * @param progress progress bar handle to update, if available. null
+         *                 otherwise
+         * @param worker   the swing worker background thread the process runs
+         *                 within, or null, if in the main thread, used to
+         *                 handle task cancellation
+         * @param source   true if source file
+         */
+        UIExtractionVisitor(File dest, ProgressHandle progress, SwingWorker<T, V> worker, boolean source) {
+            super(dest, progress, worker, source);
+        }
+
+        /**
+         * Writes content and children to disk.
+         *
+         * @param content  The root content.
+         * @param file     The TSK content file.
+         * @param dest     The disk location where the content will be written.
+         * @param progress progress bar handle to update, if available. null
+         *                 otherwise
+         * @param worker   the swing worker background thread the process runs
+         *                 within, or null, if in the main thread, used to
+         *                 handle task cancellation
+         * @param source   true if source file
+         */
+        static <T,V> void writeContent(Content content, File dest, ProgressHandle progress, SwingWorker<T, V> worker) {
+            content.accept(new UIExtractionVisitor<>(dest, progress, worker, true));
+        }
+        
+        
+        @Override
+        protected void writeFile(Content file, File dest, ProgressHandle progress, SwingWorker<T, V> worker, boolean source) throws IOException {
+            File destFile;
+            if (dest.exists()) {
+                String parent = dest.getParent();
+                String fileName = dest.getName();
+                String objIdFileName = MessageFormat.format("{0}-{1}", file.getId(), fileName);
+                destFile = new File(parent, objIdFileName);
+            } else {
+                destFile = dest;
+            }
+            
+            super.writeFile(file, destFile, progress, worker, source);   
+        }
+
+        @Override
+        protected ExtractFscContentVisitor<T, V> getChildVisitor(File childFile, ProgressHandle progress, SwingWorker<T, V> worker) {
+            return new UIExtractionVisitor(childFile, progress, worker, false);
+        }
+
+        
+        
+    }
+
     /**
      * Thread that does the actual extraction work
      */
@@ -321,8 +386,7 @@ public class ExtractActionHelper {
             // Do the extraction tasks.
             for (FileExtractionTask task : this.extractionTasks) {
                 progress.progress(Bundle.ExtractActionHelper_progress_fileExtracting(task.destination.getName()));
-
+                UIExtractionVisitor.writeContent(task.source, task.destination, null, this);
-                ContentUtils.ExtractFscContentVisitor.extract(task.source, task.destination, null, this);
             }
 
             return null;

Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java

@@ -18,6 +18,7 @@
  */
 package org.sleuthkit.autopsy.ingest;
 
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashSet;
@@ -950,18 +951,25 @@ final class IngestJobExecutor {
         currentDataSourceIngestModuleCancelled = false;
         cancelledDataSourceIngestModules.add(moduleDisplayName);
         if (usingNetBeansGUI && !jobCancelled) {
-            SwingUtilities.invokeLater(() -> {
+            try {
-                /**
+                // use invokeAndWait to ensure synchronous behavior.  
-                 * A new progress bar must be created because the cancel button
+                // See JIRA-8298 for more information.
-                 * of the previously constructed component is disabled by
+                SwingUtilities.invokeAndWait(() -> {
-                 * NetBeans when the user selects the "OK" button of the
+                    /**
-                 * cancellation confirmation dialog popped up by NetBeans when
+                     * A new progress bar must be created because the cancel
-                 * the progress bar cancel button is pressed.
+                     * button of the previously constructed component is
-                 */
+                     * disabled by NetBeans when the user selects the "OK"
-                dataSourceIngestProgressBar.finish();
+                     * button of the cancellation confirmation dialog popped up
-                dataSourceIngestProgressBar = null;
+                     * by NetBeans when the progress bar cancel button is
-                startDataSourceIngestProgressBar();
+                     * pressed.
-            });
+                     */
+                    dataSourceIngestProgressBar.finish();
+                    dataSourceIngestProgressBar = null;
+                    startDataSourceIngestProgressBar();
+                });
+            } catch (InvocationTargetException | InterruptedException ex) {
+                logger.log(Level.WARNING, "Cancellation worker cancelled.", ex);
+            }
         }
     }
 

Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java

@@ -231,7 +231,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule {
             return;
         }
 
-        aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile);
+        aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile, statusHelper);
     }
 
     /**
@@ -274,7 +274,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule {
             return;
         }
 
-        aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath);
+        aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper);
     }
 
     /**

Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED

@@ -46,9 +46,12 @@ LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts.
 LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory.
 LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory
 LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file.
+LeappFileProcessor.findTsv=Finding all Leapp ouput
 LeappFileProcessor.has.run=Leapp
 LeappFileProcessor.Leapp.cancelled=Leapp run was canceled
 LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact
 LeappFileProcessor.running.Leapp=Running Leapp
 LeappFileProcessor.starting.Leapp=Starting Leapp
+# {0} - fileName
+LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}
 LeappFileProcessor_cannotParseXml=Cannot Parse XML file.

Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java

@@ -232,7 +232,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule {
             return;
         }
 
-        iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile);
+        iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile, statusHelper);
     }
 
     /**
@@ -274,7 +274,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule {
             return;
         }
 
-        iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath);
+        iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper);
     }
 
     /**

Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java

@@ -54,6 +54,7 @@ import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.openide.util.NbBundle;
+import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.Case;
 import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
@@ -61,6 +62,7 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager;
 import org.sleuthkit.autopsy.coreutils.NetworkUtils;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
+import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
 import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
@@ -221,13 +223,19 @@ public final class LeappFileProcessor {
         "LeappFileProcessor.has.run=Leapp",
         "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled",
         "LeappFileProcessor.completed=Leapp Processing Completed",
+        "LeappFileProcessor.findTsv=Finding all Leapp ouput",
         "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"
     })
-    public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile) {
+    public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress) {
         try {
+            if (checkCancelled()) {
+                return ProcessResult.OK;
+            }
+            progress.switchToIndeterminate();
+            progress.progress(Bundle.LeappFileProcessor_findTsv());
             List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
-            processLeappFiles(LeappTsvOutputFiles, LeappFile);
+            processLeappFiles(LeappTsvOutputFiles, LeappFile, progress);
-        } catch (IOException | IngestModuleException ex) {
+        } catch (IngestModuleException ex) {
             logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
             return ProcessResult.ERROR;
         }
@@ -235,11 +243,15 @@ public final class LeappFileProcessor {
         return ProcessResult.OK;
     }
 
-    public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath) {
+    public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress) {
-
         try {
+            if (checkCancelled()) {
+                return ProcessResult.OK;
+            }
+            progress.switchToIndeterminate();
+            progress.progress(Bundle.LeappFileProcessor_findTsv());
             List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
-            processLeappFiles(LeappTsvOutputFiles, dataSource);
+            processLeappFiles(LeappTsvOutputFiles, dataSource, progress);
         } catch (IngestModuleException ex) {
             logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
             return ProcessResult.ERROR;
@@ -275,75 +287,58 @@ public final class LeappFileProcessor {
 
     }
 
-    /**
+    private boolean checkCancelled() {
-     * Process the Leapp files that were found that match the xml mapping file
+        if (this.context.dataSourceIngestIsCancelled()) {
-     *
+            logger.log(Level.INFO, "Leapp File processing module run was cancelled"); //NON-NLS
-     * @param LeappFilesToProcess List of files to process
+            return true;
-     * @param LeappImageFile      Abstract file to create artifact for
+        } else {
-     *
+            return false;
-     * @throws FileNotFoundException
-     * @throws IOException
-     */
-    private void processLeappFiles(List<String> LeappFilesToProcess, AbstractFile LeappImageFile) throws FileNotFoundException, IOException, IngestModuleException {
-        List<BlackboardArtifact> bbartifacts = new ArrayList<>();
-
-        for (String LeappFileName : LeappFilesToProcess) {
-            String fileName = FilenameUtils.getName(LeappFileName);
-            File LeappFile = new File(LeappFileName);
-            if (tsvFileAttributes.containsKey(fileName)) {
-                BlackboardArtifact.Type artifactType = null;
-                try {
-                    List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
-                    artifactType = tsvFileArtifacts.get(fileName);
-                    processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, LeappImageFile);
-                } catch (TskCoreException ex) {
-                    throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", artifactType == null ? "<null>" : artifactType.toString()), ex);
-                }
-            }
         }
-
-        if (!bbartifacts.isEmpty()) {
-            postArtifacts(bbartifacts);
-        }
-
     }
 
     /**
      * Process the Leapp files that were found that match the xml mapping file
      *
-     * @param LeappFilesToProcess List of files to process
+     * @param LeappFilesToProcess List of files to process.
      * @param dataSource          The data source.
+     * @param progress            Means of updating progress in UI.
      *
      * @throws FileNotFoundException
      * @throws IOException
      */
-    private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource) throws IngestModuleException {
+    @Messages({
-        List<BlackboardArtifact> bbartifacts = new ArrayList<>();
+        "# {0} - fileName",
+        "LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}"
+    })
+    private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress) throws IngestModuleException {
+        progress.switchToDeterminate(LeappFilesToProcess.size());
 
-        for (String LeappFileName : LeappFilesToProcess) {
+        for (int i = 0; i < LeappFilesToProcess.size(); i++) {
+            if (checkCancelled()) {
+                return;
+            }
+
+            String LeappFileName = LeappFilesToProcess.get(i);
             String fileName = FilenameUtils.getName(LeappFileName);
+            progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i);
+
             File LeappFile = new File(LeappFileName);
             if (tsvFileAttributes.containsKey(fileName)) {
                 List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
                 BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(fileName);
 
                 try {
-                    processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource);
+                    processFile(LeappFile, attrList, fileName, artifactType, dataSource);
                 } catch (TskCoreException | IOException ex) {
                     logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex);
                 }
             }
 
         }
-
-        if (!bbartifacts.isEmpty()) {
-            postArtifacts(bbartifacts);
-        }
-
     }
 
-    private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName, BlackboardArtifact.Type artifactType,
+    private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName,
-            List<BlackboardArtifact> bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException,
+            BlackboardArtifact.Type artifactType, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException,
             TskCoreException {
 
         String trackpointSegmentName = null;
@@ -358,6 +353,8 @@ public final class LeappFileProcessor {
             return;
         }
 
+        List<BlackboardArtifact> bbartifacts = new ArrayList<>();
+
         // based on https://stackoverflow.com/questions/56921465/jackson-csv-schema-for-array
         try (MappingIterator<List<String>> iterator = new CsvMapper()
                 .enable(CsvParser.Feature.WRAP_AS_ARRAY)
@@ -418,6 +415,9 @@ public final class LeappFileProcessor {
             throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
         }
 
+        if (!bbartifacts.isEmpty()) {
+            postArtifacts(bbartifacts);
+        }
     }
 
     @NbBundle.Messages({

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java

@@ -165,7 +165,7 @@ public class KeywordList {
     /**
      * Gets the keywords included in the list
      *
-     * @return A colleciton of Keyword objects.
+     * @return A collection of Keyword objects.
      */
     public List<Keyword> getKeywords() {
         return keywords;

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java

@@ -57,6 +57,8 @@ class QueryResults {
     private final KeywordSearchQuery query;
     private final Map<Keyword, List<KeywordHit>> results = new HashMap<>();
     
+    private static final int MAX_INBOX_NOTIFICATIONS_PER_KW_TERM = 20;
+
     /**
      * Constructs a object that stores and processes the results of a keyword
      * search query. Processing includes adding keyword hit artifacts to the
@@ -142,6 +144,8 @@ class QueryResults {
      */
     void process(SwingWorker<?, ?> worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) {
         final Collection<BlackboardArtifact> hitArtifacts = new ArrayList<>();
+        
+        int notificationCount = 0;
         for (final Keyword keyword : getKeywords()) {
             /*
              * Cancellation check.
@@ -200,8 +204,12 @@ class QueryResults {
                      */
                     if (null != artifact) {
                         hitArtifacts.add(artifact);
-                        if (notifyInbox) {
+                        if (notifyInbox && notificationCount < MAX_INBOX_NOTIFICATIONS_PER_KW_TERM) {
+                            // only send ingest inbox messages for the first MAX_INBOX_NOTIFICATIONS_PER_KW_TERM hits
+                            // for every KW term (per ingest job, aka data source). Otherwise we can have a situation
+                            // where we send tens of thousands of notifications. 
                             try {
+                                notificationCount++;
                                 writeSingleFileInboxMessage(artifact, content);
                             } catch (TskCoreException ex) {
                                 logger.log(Level.SEVERE, "Error sending message to ingest messages inbox", ex); //NON-NLS

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -30,6 +30,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.logging.Level;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -83,6 +85,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
     private Blackboard blackboard;
     private CommunicationArtifactsHelper communicationArtifactsHelper;
     
+    // A cache of custom attributes for the VcardParser unique to each ingest run, but consistent across threads.
+    private static ConcurrentMap<String, BlackboardAttribute.Type> customAttributeCache = new ConcurrentHashMap<>();
+    private static Object customAttributeCacheLock = new Object();
+    
     private static final int MBOX_SIZE_TO_SPLIT = 1048576000;
     private Case currentCase;
 
@@ -96,6 +102,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
     @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."})
     public void startUp(IngestJobContext context) throws IngestModuleException {
         this.context = context;
+        
+        synchronized(customAttributeCacheLock) {
+            if (!customAttributeCache.isEmpty()) {
+                customAttributeCache.clear();
+            }
+        }
+        
         try {
             currentCase = Case.getCurrentCaseThrows();
             fileManager = Case.getCurrentCaseThrows().getServices().getFileManager();
@@ -441,7 +454,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
     })
     private ProcessResult processVcard(AbstractFile abstractFile) {
         try {
-            VcardParser parser = new VcardParser(currentCase, context);
+            VcardParser parser = new VcardParser(currentCase, context, customAttributeCache);
             parser.parse(abstractFile);
         } catch (IOException | NoCurrentCaseException ex) {
             logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS
@@ -912,7 +925,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
 
     @Override
     public void shutDown() {
-        // nothing to shut down
+        synchronized(customAttributeCacheLock) {
+            if (!customAttributeCache.isEmpty()) {
+                customAttributeCache.clear();
+            }
+        }
     }
 
 }

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java

@@ -40,6 +40,7 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
 import java.util.logging.Level;
 import org.apache.commons.lang3.StringUtils;
 import org.openide.util.NbBundle;
@@ -98,16 +99,22 @@ final class VcardParser {
     private final Blackboard blackboard;
     private final Case currentCase;
     private final SleuthkitCase tskCase;
+    /**
+     * A custom attribute cache provided to every VcardParser from the
+     * ThunderbirdMboxFileIngestModule, but unique to one ingest run.
+     */
+    private final ConcurrentMap<String, BlackboardAttribute.Type> customAttributeCache;
     
     /**
      * Create a VcardParser object.
      */
-    VcardParser(Case currentCase, IngestJobContext context) {
+    VcardParser(Case currentCase, IngestJobContext context, ConcurrentMap<String, BlackboardAttribute.Type> customAttributeCache) {
         this.context = context;
         this.currentCase = currentCase;
         tskCase = currentCase.getSleuthkitCase();
         blackboard = tskCase.getBlackboard();
         fileManager = currentCase.getServices().getFileManager();
+        this.customAttributeCache = customAttributeCache;
     }
 
     /**
@@ -422,23 +429,26 @@ final class VcardParser {
                     attributeTypeName = "TSK_PHONE_NUMBER_" + splitType;
                 }
                 
-                try {
+                final String finalAttrTypeName = attributeTypeName;
-                    BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName);
-                    if (attributeType == null) {
-                        try{
-                            // Add this attribute type to the case database.
-                            attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName,
-                                    BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING,
-                                    String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase())));
 
-                            ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes);
+                // handled in computeIfAbsent to remove concurrency issues when adding to this concurrent hashmap.
-                        }catch (BlackboardException ex) {
+                BlackboardAttribute.Type attributeType
-                            logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
+                        = this.customAttributeCache.computeIfAbsent(finalAttrTypeName, k -> {
-                        }
+                            try {
-                    }
+                                // Add this attribute type to the case database.
+                                return tskCase.getBlackboard().getOrAddAttributeType(finalAttrTypeName,
+                                        BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING,
+                                        String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase())));
 
-                } catch (TskCoreException ex) {
+                            } catch (BlackboardException ex) {
-                    logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
+                                VcardParser.logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).",
+                                        finalAttrTypeName, abstractFile.getName(), abstractFile.getId()), ex);
+                                return null;
+                            }
+                        });
+
+                if (attributeType != null) {
+                    ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes);
                 }
             }
         }
@@ -469,30 +479,37 @@ final class VcardParser {
             * ez-vcard. Therefore, we must read them manually
             * ourselves.
             */
-           List<String> splitEmailTypes = Arrays.asList(
+            List<String> splitEmailTypes = Arrays.asList(
-                   type.getValue().toUpperCase().replaceAll("\\s+","").split(","));
+                    type.getValue().toUpperCase().replaceAll("\\s+", "").split(","));
 
-           if (splitEmailTypes.size() > 0) {
+            if (splitEmailTypes.size() > 0) {
-               String splitType = splitEmailTypes.get(0);
+                String splitType = splitEmailTypes.get(0);
-               String attributeTypeName = "TSK_EMAIL_" + splitType;
+                String attributeTypeName = "TSK_EMAIL_" + splitType;
-               if(splitType.isEmpty()) {
+                if (splitType.isEmpty()) {
-                   attributeTypeName = "TSK_EMAIL";
+                    attributeTypeName = "TSK_EMAIL";
-               }
+                }
-               try {
+
-                   BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName);
+                final String finalAttributeTypeName = attributeTypeName;
-                   if (attributeType == null) {
+
-                       // Add this attribute type to the case database.
+                BlackboardAttribute.Type attributeType
-                       attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName, 
+                        = this.customAttributeCache.computeIfAbsent(finalAttributeTypeName, k -> {
-                               BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, 
+                            try {
-                               String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase())));
+                                // Add this attribute type to the case database.
-                   }
+                                return tskCase.getBlackboard().getOrAddAttributeType(finalAttributeTypeName,
-                   ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes);
+                                        BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING,
-               } catch (TskCoreException ex) {
+                                        String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase())));
-                   logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
+                            } catch (BlackboardException ex) {
-               } catch (BlackboardException ex) {
+                                logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).",
-                   logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
+                                        finalAttributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
-               }
+                            }
-           }  
+
+                            return null;
+                        });
+
+                if (attributeType != null) {
+                    ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes);
+                }
+            }
         }
     }