diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index 6def78b83d..0766316e67 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -392,85 +392,69 @@ public final class ContentUtils { public static void extract(Content cntnt, java.io.File dest, ProgressHandle progress, SwingWorker worker) { cntnt.accept(new ExtractFscContentVisitor<>(dest, progress, worker, true)); } + + /** + * Base method writing a file to disk. + * + * @param file The TSK content file. + * @param dest The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @param source true if source file + * + * @throws IOException + */ + protected void writeFile(Content file, java.io.File dest, ProgressHandle progress, SwingWorker worker, boolean source) throws IOException { + ContentUtils.writeToFile(file, dest, progress, worker, source); + } - @Override - public Void visit(File file) { + /** + * Visits a TSK content file and writes that file to disk. + * @param file The file to be written. + * @param fileType The file type (i.e. "derived file") for error logging. + * @return null. + */ + protected Void visitFile(Content file, String fileType) { try { - ContentUtils.writeToFile(file, dest, progress, worker, source); + writeFile(file, dest, progress, worker, source); } catch (ReadContentInputStreamException ex) { logger.log(Level.WARNING, String.format("Error reading file '%s' (id=%d).", file.getName(), file.getId()), ex); //NON-NLS } catch (IOException ex) { logger.log(Level.SEVERE, - String.format("Error extracting file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS + String.format("Error extracting %s '%s' (id=%d) to '%s'.", + fileType, file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS } return null; } + + @Override + public Void visit(File file) { + return visitFile(file, "file"); + } @Override public Void visit(LayoutFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting unallocated content file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "unallocated content file"); } @Override public Void visit(DerivedFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting derived file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "derived file"); } @Override public Void visit(LocalFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting local file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "local file"); } @Override public Void visit(SlackFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting slack file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "slack file"); } @Override @@ -493,6 +477,20 @@ public final class ContentUtils { + content.getName(); return new java.io.File(path); } + + /** + * Returns a visitor to visit any child content. + * @param childFile The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @return + */ + protected ExtractFscContentVisitor getChildVisitor(java.io.File childFile, ProgressHandle progress, SwingWorker worker) { + return new ExtractFscContentVisitor<>(childFile, progress, worker, false); + } public Void visitDir(AbstractFile dir) { @@ -509,8 +507,7 @@ public final class ContentUtils { for (Content child : dir.getChildren()) { if (child instanceof AbstractFile) { //ensure the directory's artifact children are ignored java.io.File childFile = getFsContentDest(child); - ExtractFscContentVisitor childVisitor - = new ExtractFscContentVisitor<>(childFile, progress, worker, false); + ExtractFscContentVisitor childVisitor = getChildVisitor(childFile, progress, worker); // If this is the source directory of an extract it // will have a progress and worker, and will keep track // of the progress bar's progress diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java index 0efe3bdcfb..6a585d724f 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java @@ -43,6 +43,8 @@ public final class ExtractAction extends AbstractAction { } return instance; } + + private final ExtractActionHelper extractor = new ExtractActionHelper(); /** * Private constructor for the action. @@ -61,7 +63,6 @@ public final class ExtractAction extends AbstractAction { public void actionPerformed(ActionEvent e) { Lookup lookup = Utilities.actionsGlobalContext(); Collection selectedFiles =lookup.lookupAll(AbstractFile.class); - ExtractActionHelper extractor = new ExtractActionHelper(); extractor.extract(e, selectedFiles); } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java index 5a28bd9c12..386240ea86 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java @@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.directorytree.actionhelpers; import java.awt.Component; import java.awt.event.ActionEvent; import java.io.File; +import java.io.IOException; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -42,8 +44,10 @@ import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.datamodel.ContentUtils.ExtractFscContentVisitor; import org.sleuthkit.autopsy.guiutils.JFileChooserFactory; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; /** * Helper class for methods needed by actions which extract files. @@ -52,7 +56,7 @@ public class ExtractActionHelper { private final Logger logger = Logger.getLogger(ExtractActionHelper.class.getName()); private String userDefinedExportPath; - + private final JFileChooserFactory extractFileHelper = new JFileChooserFactory(); private final JFileChooserFactory extractFilesHelper = new JFileChooserFactory(); @@ -68,12 +72,7 @@ public class ExtractActionHelper { if (selectedFiles.size() > 1) { extractFiles(event, selectedFiles); } else if (selectedFiles.size() == 1) { - AbstractFile source = selectedFiles.iterator().next(); - if (source.isDir()) { - extractFiles(event, selectedFiles); - } else { - extractFile(event, selectedFiles.iterator().next()); - } + extractFile(event, selectedFiles.iterator().next()); } } @@ -83,7 +82,11 @@ public class ExtractActionHelper { * @param event * @param selectedFile Selected file */ - @NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available."}) + @NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available.", + "ExtractActionHelper.extractOverwrite.title=Export to csv file", + "# {0} - fileName", + "ExtractActionHelper.extractOverwrite.msg=A file already exists at {0}. Do you want to overwrite the existing file?" + }) private void extractFile(ActionEvent event, AbstractFile selectedFile) { Case openCase; try { @@ -269,6 +272,68 @@ public class ExtractActionHelper { } } + /** + * A file content extraction visitor that handles for the UI designed to + * handle file name conflicts by appending the object id to the file name. + */ + private static class UIExtractionVisitor extends ExtractFscContentVisitor { + + /** + * @param file The TSK content file. + * @param dest The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @param source true if source file + */ + UIExtractionVisitor(File dest, ProgressHandle progress, SwingWorker worker, boolean source) { + super(dest, progress, worker, source); + } + + /** + * Writes content and children to disk. + * + * @param content The root content. + * @param file The TSK content file. + * @param dest The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @param source true if source file + */ + static void writeContent(Content content, File dest, ProgressHandle progress, SwingWorker worker) { + content.accept(new UIExtractionVisitor<>(dest, progress, worker, true)); + } + + + @Override + protected void writeFile(Content file, File dest, ProgressHandle progress, SwingWorker worker, boolean source) throws IOException { + File destFile; + if (dest.exists()) { + String parent = dest.getParent(); + String fileName = dest.getName(); + String objIdFileName = MessageFormat.format("{0}-{1}", file.getId(), fileName); + destFile = new File(parent, objIdFileName); + } else { + destFile = dest; + } + + super.writeFile(file, destFile, progress, worker, source); + } + + @Override + protected ExtractFscContentVisitor getChildVisitor(File childFile, ProgressHandle progress, SwingWorker worker) { + return new UIExtractionVisitor(childFile, progress, worker, false); + } + + + + } + /** * Thread that does the actual extraction work */ @@ -321,8 +386,7 @@ public class ExtractActionHelper { // Do the extraction tasks. for (FileExtractionTask task : this.extractionTasks) { progress.progress(Bundle.ExtractActionHelper_progress_fileExtracting(task.destination.getName())); - - ContentUtils.ExtractFscContentVisitor.extract(task.source, task.destination, null, this); + UIExtractionVisitor.writeContent(task.source, task.destination, null, this); } return null; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 44b28c415a..f665c4bcf3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; @@ -950,18 +951,25 @@ final class IngestJobExecutor { currentDataSourceIngestModuleCancelled = false; cancelledDataSourceIngestModules.add(moduleDisplayName); if (usingNetBeansGUI && !jobCancelled) { - SwingUtilities.invokeLater(() -> { - /** - * A new progress bar must be created because the cancel button - * of the previously constructed component is disabled by - * NetBeans when the user selects the "OK" button of the - * cancellation confirmation dialog popped up by NetBeans when - * the progress bar cancel button is pressed. - */ - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - startDataSourceIngestProgressBar(); - }); + try { + // use invokeAndWait to ensure synchronous behavior. + // See JIRA-8298 for more information. + SwingUtilities.invokeAndWait(() -> { + /** + * A new progress bar must be created because the cancel + * button of the previously constructed component is + * disabled by NetBeans when the user selects the "OK" + * button of the cancellation confirmation dialog popped up + * by NetBeans when the progress bar cancel button is + * pressed. + */ + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + startDataSourceIngestProgressBar(); + }); + } catch (InvocationTargetException | InterruptedException ex) { + logger.log(Level.WARNING, "Cancellation worker cancelled.", ex); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java index c998128d63..3cfe4b954c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java @@ -231,7 +231,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile); + aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile, statusHelper); } /** @@ -274,7 +274,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath); + aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED index a1d45fa3ad..d861b2fc29 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED @@ -46,9 +46,12 @@ LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts. LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory. LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file. +LeappFileProcessor.findTsv=Finding all Leapp ouput LeappFileProcessor.has.run=Leapp LeappFileProcessor.Leapp.cancelled=Leapp run was canceled LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact LeappFileProcessor.running.Leapp=Running Leapp LeappFileProcessor.starting.Leapp=Starting Leapp +# {0} - fileName +LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0} LeappFileProcessor_cannotParseXml=Cannot Parse XML file. diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java index 5e2365d38b..b46982dfa6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java @@ -232,7 +232,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile); + iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile, statusHelper); } /** @@ -274,7 +274,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath); + iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 8b500820d2..1e4e30a944 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -54,6 +54,7 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -61,6 +62,7 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; @@ -221,13 +223,19 @@ public final class LeappFileProcessor { "LeappFileProcessor.has.run=Leapp", "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled", "LeappFileProcessor.completed=Leapp Processing Completed", + "LeappFileProcessor.findTsv=Finding all Leapp ouput", "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory" }) - public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile) { + public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress) { try { + if (checkCancelled()) { + return ProcessResult.OK; + } + progress.switchToIndeterminate(); + progress.progress(Bundle.LeappFileProcessor_findTsv()); List LeappTsvOutputFiles = findTsvFiles(moduleOutputPath); - processLeappFiles(LeappTsvOutputFiles, LeappFile); - } catch (IOException | IngestModuleException ex) { + processLeappFiles(LeappTsvOutputFiles, LeappFile, progress); + } catch (IngestModuleException ex) { logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS return ProcessResult.ERROR; } @@ -235,11 +243,15 @@ public final class LeappFileProcessor { return ProcessResult.OK; } - public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath) { - + public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress) { try { + if (checkCancelled()) { + return ProcessResult.OK; + } + progress.switchToIndeterminate(); + progress.progress(Bundle.LeappFileProcessor_findTsv()); List LeappTsvOutputFiles = findTsvFiles(moduleOutputPath); - processLeappFiles(LeappTsvOutputFiles, dataSource); + processLeappFiles(LeappTsvOutputFiles, dataSource, progress); } catch (IngestModuleException ex) { logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS return ProcessResult.ERROR; @@ -275,75 +287,58 @@ public final class LeappFileProcessor { } - /** - * Process the Leapp files that were found that match the xml mapping file - * - * @param LeappFilesToProcess List of files to process - * @param LeappImageFile Abstract file to create artifact for - * - * @throws FileNotFoundException - * @throws IOException - */ - private void processLeappFiles(List LeappFilesToProcess, AbstractFile LeappImageFile) throws FileNotFoundException, IOException, IngestModuleException { - List bbartifacts = new ArrayList<>(); - - for (String LeappFileName : LeappFilesToProcess) { - String fileName = FilenameUtils.getName(LeappFileName); - File LeappFile = new File(LeappFileName); - if (tsvFileAttributes.containsKey(fileName)) { - BlackboardArtifact.Type artifactType = null; - try { - List attrList = tsvFileAttributes.get(fileName); - artifactType = tsvFileArtifacts.get(fileName); - processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, LeappImageFile); - } catch (TskCoreException ex) { - throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", artifactType == null ? "" : artifactType.toString()), ex); - } - } + private boolean checkCancelled() { + if (this.context.dataSourceIngestIsCancelled()) { + logger.log(Level.INFO, "Leapp File processing module run was cancelled"); //NON-NLS + return true; + } else { + return false; } - - if (!bbartifacts.isEmpty()) { - postArtifacts(bbartifacts); - } - } /** * Process the Leapp files that were found that match the xml mapping file * - * @param LeappFilesToProcess List of files to process + * @param LeappFilesToProcess List of files to process. * @param dataSource The data source. + * @param progress Means of updating progress in UI. * * @throws FileNotFoundException * @throws IOException */ - private void processLeappFiles(List LeappFilesToProcess, Content dataSource) throws IngestModuleException { - List bbartifacts = new ArrayList<>(); + @Messages({ + "# {0} - fileName", + "LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}" + }) + private void processLeappFiles(List LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress) throws IngestModuleException { + progress.switchToDeterminate(LeappFilesToProcess.size()); - for (String LeappFileName : LeappFilesToProcess) { + for (int i = 0; i < LeappFilesToProcess.size(); i++) { + if (checkCancelled()) { + return; + } + + String LeappFileName = LeappFilesToProcess.get(i); String fileName = FilenameUtils.getName(LeappFileName); + progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i); + File LeappFile = new File(LeappFileName); if (tsvFileAttributes.containsKey(fileName)) { List attrList = tsvFileAttributes.get(fileName); BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(fileName); try { - processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource); + processFile(LeappFile, attrList, fileName, artifactType, dataSource); } catch (TskCoreException | IOException ex) { logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex); } } } - - if (!bbartifacts.isEmpty()) { - postArtifacts(bbartifacts); - } - } - private void processFile(File LeappFile, List attrList, String fileName, BlackboardArtifact.Type artifactType, - List bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException, + private void processFile(File LeappFile, List attrList, String fileName, + BlackboardArtifact.Type artifactType, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException, TskCoreException { String trackpointSegmentName = null; @@ -358,6 +353,8 @@ public final class LeappFileProcessor { return; } + List bbartifacts = new ArrayList<>(); + // based on https://stackoverflow.com/questions/56921465/jackson-csv-schema-for-array try (MappingIterator> iterator = new CsvMapper() .enable(CsvParser.Feature.WRAP_AS_ARRAY) @@ -418,6 +415,9 @@ public final class LeappFileProcessor { throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS } + if (!bbartifacts.isEmpty()) { + postArtifacts(bbartifacts); + } } @NbBundle.Messages({ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java index f2db250baa..ea2b150d7f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java @@ -165,7 +165,7 @@ public class KeywordList { /** * Gets the keywords included in the list * - * @return A colleciton of Keyword objects. + * @return A collection of Keyword objects. */ public List getKeywords() { return keywords; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 5ac61eb946..1c6bfeba1f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -56,6 +56,8 @@ class QueryResults { private static final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); private final KeywordSearchQuery query; private final Map> results = new HashMap<>(); + + private static final int MAX_INBOX_NOTIFICATIONS_PER_KW_TERM = 20; /** * Constructs a object that stores and processes the results of a keyword @@ -142,6 +144,8 @@ class QueryResults { */ void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { final Collection hitArtifacts = new ArrayList<>(); + + int notificationCount = 0; for (final Keyword keyword : getKeywords()) { /* * Cancellation check. @@ -150,7 +154,7 @@ class QueryResults { logger.log(Level.INFO, "Processing cancelled, exiting before processing search term {0}", keyword.getSearchTerm()); //NON-NLS return; } - + /* * Reduce the hits for this keyword to one hit per text source * object so that only one hit artifact is generated per text source @@ -200,8 +204,12 @@ class QueryResults { */ if (null != artifact) { hitArtifacts.add(artifact); - if (notifyInbox) { + if (notifyInbox && notificationCount < MAX_INBOX_NOTIFICATIONS_PER_KW_TERM) { + // only send ingest inbox messages for the first MAX_INBOX_NOTIFICATIONS_PER_KW_TERM hits + // for every KW term (per ingest job, aka data source). Otherwise we can have a situation + // where we send tens of thousands of notifications. try { + notificationCount++; writeSingleFileInboxMessage(artifact, content); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error sending message to ingest messages inbox", ex); //NON-NLS diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index ea95afa9f6..13ca786c3b 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -30,6 +30,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -82,7 +84,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private IngestJobContext context; private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; - + + // A cache of custom attributes for the VcardParser unique to each ingest run, but consistent across threads. + private static ConcurrentMap customAttributeCache = new ConcurrentHashMap<>(); + private static Object customAttributeCacheLock = new Object(); + private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; @@ -96,6 +102,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + + synchronized(customAttributeCacheLock) { + if (!customAttributeCache.isEmpty()) { + customAttributeCache.clear(); + } + } + try { currentCase = Case.getCurrentCaseThrows(); fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); @@ -441,7 +454,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { }) private ProcessResult processVcard(AbstractFile abstractFile) { try { - VcardParser parser = new VcardParser(currentCase, context); + VcardParser parser = new VcardParser(currentCase, context, customAttributeCache); parser.parse(abstractFile); } catch (IOException | NoCurrentCaseException ex) { logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS @@ -912,7 +925,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { @Override public void shutDown() { - // nothing to shut down + synchronized(customAttributeCacheLock) { + if (!customAttributeCache.isEmpty()) { + customAttributeCache.clear(); + } + } } } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index 1c2bab5fef..306843f025 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -40,6 +40,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentMap; import java.util.logging.Level; import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; @@ -98,16 +99,22 @@ final class VcardParser { private final Blackboard blackboard; private final Case currentCase; private final SleuthkitCase tskCase; + /** + * A custom attribute cache provided to every VcardParser from the + * ThunderbirdMboxFileIngestModule, but unique to one ingest run. + */ + private final ConcurrentMap customAttributeCache; /** * Create a VcardParser object. */ - VcardParser(Case currentCase, IngestJobContext context) { + VcardParser(Case currentCase, IngestJobContext context, ConcurrentMap customAttributeCache) { this.context = context; this.currentCase = currentCase; tskCase = currentCase.getSleuthkitCase(); blackboard = tskCase.getBlackboard(); fileManager = currentCase.getServices().getFileManager(); + this.customAttributeCache = customAttributeCache; } /** @@ -421,26 +428,29 @@ final class VcardParser { if (splitType != null && !splitType.isEmpty()) { attributeTypeName = "TSK_PHONE_NUMBER_" + splitType; } + + final String finalAttrTypeName = attributeTypeName; - try { - BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName); - if (attributeType == null) { - try{ - // Add this attribute type to the case database. - attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName, - BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, - String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase()))); - - ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); - }catch (BlackboardException ex) { - logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } - } - - } catch (TskCoreException ex) { - logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + // handled in computeIfAbsent to remove concurrency issues when adding to this concurrent hashmap. + BlackboardAttribute.Type attributeType + = this.customAttributeCache.computeIfAbsent(finalAttrTypeName, k -> { + try { + // Add this attribute type to the case database. + return tskCase.getBlackboard().getOrAddAttributeType(finalAttrTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + + } catch (BlackboardException ex) { + VcardParser.logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", + finalAttrTypeName, abstractFile.getName(), abstractFile.getId()), ex); + return null; + } + }); + + if (attributeType != null) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); } - } + } } } @@ -469,30 +479,37 @@ final class VcardParser { * ez-vcard. Therefore, we must read them manually * ourselves. */ - List splitEmailTypes = Arrays.asList( - type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); + List splitEmailTypes = Arrays.asList( + type.getValue().toUpperCase().replaceAll("\\s+", "").split(",")); - if (splitEmailTypes.size() > 0) { - String splitType = splitEmailTypes.get(0); - String attributeTypeName = "TSK_EMAIL_" + splitType; - if(splitType.isEmpty()) { - attributeTypeName = "TSK_EMAIL"; - } - try { - BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName); - if (attributeType == null) { - // Add this attribute type to the case database. - attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName, - BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, - String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); - } - ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } catch (BlackboardException ex) { - logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } - } + if (splitEmailTypes.size() > 0) { + String splitType = splitEmailTypes.get(0); + String attributeTypeName = "TSK_EMAIL_" + splitType; + if (splitType.isEmpty()) { + attributeTypeName = "TSK_EMAIL"; + } + + final String finalAttributeTypeName = attributeTypeName; + + BlackboardAttribute.Type attributeType + = this.customAttributeCache.computeIfAbsent(finalAttributeTypeName, k -> { + try { + // Add this attribute type to the case database. + return tskCase.getBlackboard().getOrAddAttributeType(finalAttributeTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + } catch (BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", + finalAttributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } + + return null; + }); + + if (attributeType != null) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); + } + } } }