From 1dd89489b1f9e437fbce94a9c1cdb13a29f704f8 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 24 Jan 2022 11:27:17 -0500 Subject: [PATCH 01/12] progress and cancellation --- .../ALeappAnalyzerIngestModule.java | 4 +- .../ILeappAnalyzerIngestModule.java | 4 +- .../leappanalyzers/LeappFileProcessor.java | 98 +++++++++---------- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java index c998128d63..3cfe4b954c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java @@ -231,7 +231,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile); + aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile, statusHelper); } /** @@ -274,7 +274,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath); + aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java index 5e2365d38b..b46982dfa6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java @@ -232,7 +232,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile); + iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile, statusHelper); } /** @@ -274,7 +274,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { return; } - iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath); + iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 8b500820d2..0422ef8a38 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -54,6 +54,7 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; +import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -61,6 +62,7 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; @@ -221,13 +223,19 @@ public final class LeappFileProcessor { "LeappFileProcessor.has.run=Leapp", "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled", "LeappFileProcessor.completed=Leapp Processing Completed", + "LeappFileProcessor.findTsv=Finding all Leapp ouput", "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory" }) - public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile) { + public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress) { try { + if (checkCancelled()) { + return ProcessResult.OK; + } + progress.switchToIndeterminate(); + progress.progress(Bundle.LeappFileProcessor_findTsv()); List LeappTsvOutputFiles = findTsvFiles(moduleOutputPath); - processLeappFiles(LeappTsvOutputFiles, LeappFile); - } catch (IOException | IngestModuleException ex) { + processLeappFiles(LeappTsvOutputFiles, LeappFile, progress); + } catch (IngestModuleException ex) { logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS return ProcessResult.ERROR; } @@ -235,11 +243,15 @@ public final class LeappFileProcessor { return ProcessResult.OK; } - public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath) { - + public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress) { try { + if (checkCancelled()) { + return ProcessResult.OK; + } + progress.switchToIndeterminate(); + progress.progress(Bundle.LeappFileProcessor_findTsv()); List LeappTsvOutputFiles = findTsvFiles(moduleOutputPath); - processLeappFiles(LeappTsvOutputFiles, dataSource); + processLeappFiles(LeappTsvOutputFiles, dataSource, progress); } catch (IngestModuleException ex) { logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS return ProcessResult.ERROR; @@ -275,75 +287,58 @@ public final class LeappFileProcessor { } - /** - * Process the Leapp files that were found that match the xml mapping file - * - * @param LeappFilesToProcess List of files to process - * @param LeappImageFile Abstract file to create artifact for - * - * @throws FileNotFoundException - * @throws IOException - */ - private void processLeappFiles(List LeappFilesToProcess, AbstractFile LeappImageFile) throws FileNotFoundException, IOException, IngestModuleException { - List bbartifacts = new ArrayList<>(); - - for (String LeappFileName : LeappFilesToProcess) { - String fileName = FilenameUtils.getName(LeappFileName); - File LeappFile = new File(LeappFileName); - if (tsvFileAttributes.containsKey(fileName)) { - BlackboardArtifact.Type artifactType = null; - try { - List attrList = tsvFileAttributes.get(fileName); - artifactType = tsvFileArtifacts.get(fileName); - processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, LeappImageFile); - } catch (TskCoreException ex) { - throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", artifactType == null ? "" : artifactType.toString()), ex); - } - } + private boolean checkCancelled() { + if (this.context.dataSourceIngestIsCancelled()) { + logger.log(Level.INFO, "Leapp File processing module run was cancelled"); //NON-NLS + return true; + } else { + return false; } - - if (!bbartifacts.isEmpty()) { - postArtifacts(bbartifacts); - } - } /** * Process the Leapp files that were found that match the xml mapping file * - * @param LeappFilesToProcess List of files to process + * @param LeappFilesToProcess List of files to process. * @param dataSource The data source. + * @param progress Means of updating progress in UI. * * @throws FileNotFoundException * @throws IOException */ - private void processLeappFiles(List LeappFilesToProcess, Content dataSource) throws IngestModuleException { - List bbartifacts = new ArrayList<>(); - - for (String LeappFileName : LeappFilesToProcess) { + @Messages({ + "# {0} - fileName", + "LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}" + }) + private void processLeappFiles(List LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress) throws IngestModuleException { + progress.switchToDeterminate(LeappFilesToProcess.size()); + + for (int i = 0; i < LeappFilesToProcess.size(); i++) { + if (checkCancelled()) { + return; + } + + String LeappFileName = LeappFilesToProcess.get(i); String fileName = FilenameUtils.getName(LeappFileName); + progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i); + File LeappFile = new File(LeappFileName); if (tsvFileAttributes.containsKey(fileName)) { List attrList = tsvFileAttributes.get(fileName); BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(fileName); try { - processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource); + processFile(LeappFile, attrList, fileName, artifactType, dataSource); } catch (TskCoreException | IOException ex) { logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex); } } } - - if (!bbartifacts.isEmpty()) { - postArtifacts(bbartifacts); - } - } - private void processFile(File LeappFile, List attrList, String fileName, BlackboardArtifact.Type artifactType, - List bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException, + private void processFile(File LeappFile, List attrList, String fileName, + BlackboardArtifact.Type artifactType, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException, TskCoreException { String trackpointSegmentName = null; @@ -358,6 +353,8 @@ public final class LeappFileProcessor { return; } + List bbartifacts = new ArrayList<>(); + // based on https://stackoverflow.com/questions/56921465/jackson-csv-schema-for-array try (MappingIterator> iterator = new CsvMapper() .enable(CsvParser.Feature.WRAP_AS_ARRAY) @@ -418,6 +415,9 @@ public final class LeappFileProcessor { throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS } + if (!bbartifacts.isEmpty()) { + postArtifacts(bbartifacts); + } } @NbBundle.Messages({ From 647f10c23ccc7ddb43fe1c59dd652219dd54afbc Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 24 Jan 2022 11:52:56 -0500 Subject: [PATCH 02/12] messages and formatting --- .../autopsy/modules/leappanalyzers/Bundle.properties-MERGED | 3 +++ .../autopsy/modules/leappanalyzers/LeappFileProcessor.java | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED index a1d45fa3ad..d861b2fc29 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED @@ -46,9 +46,12 @@ LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts. LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory. LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file. +LeappFileProcessor.findTsv=Finding all Leapp ouput LeappFileProcessor.has.run=Leapp LeappFileProcessor.Leapp.cancelled=Leapp run was canceled LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact LeappFileProcessor.running.Leapp=Running Leapp LeappFileProcessor.starting.Leapp=Starting Leapp +# {0} - fileName +LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0} LeappFileProcessor_cannotParseXml=Cannot Parse XML file. diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 0422ef8a38..1e4e30a944 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -312,16 +312,16 @@ public final class LeappFileProcessor { }) private void processLeappFiles(List LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress) throws IngestModuleException { progress.switchToDeterminate(LeappFilesToProcess.size()); - + for (int i = 0; i < LeappFilesToProcess.size(); i++) { if (checkCancelled()) { return; } - + String LeappFileName = LeappFilesToProcess.get(i); String fileName = FilenameUtils.getName(LeappFileName); progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i); - + File LeappFile = new File(LeappFileName); if (tsvFileAttributes.containsKey(fileName)) { List attrList = tsvFileAttributes.get(fileName); From 23eb75b12b440bbc140964f8250f97e862b65b89 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 24 Jan 2022 20:17:12 -0500 Subject: [PATCH 03/12] custom attr cache --- .../ThunderbirdMboxFileIngestModule.java | 7 +- .../thunderbirdparser/VcardParser.java | 100 ++++++++++-------- 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index ea95afa9f6..0000847c63 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -30,6 +30,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -82,6 +83,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private IngestJobContext context; private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; + + // A cache of custom attributes for the VcardParser unique to each ingest run. + private Map customAttributeCache; private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; @@ -96,6 +100,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + this.customAttributeCache = new ConcurrentHashMap<>(); try { currentCase = Case.getCurrentCaseThrows(); fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); @@ -441,7 +446,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { }) private ProcessResult processVcard(AbstractFile abstractFile) { try { - VcardParser parser = new VcardParser(currentCase, context); + VcardParser parser = new VcardParser(currentCase, context, customAttributeCache); parser.parse(abstractFile); } catch (IOException | NoCurrentCaseException ex) { logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index 1c2bab5fef..6e3ff0082b 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -98,16 +98,22 @@ final class VcardParser { private final Blackboard blackboard; private final Case currentCase; private final SleuthkitCase tskCase; + /** + * A custom attribute cache provided to every VcardParser from the + * ThunderbirdMboxFileIngestModule, but unique to one ingest run. + */ + private final Map customAttributeCache; /** * Create a VcardParser object. */ - VcardParser(Case currentCase, IngestJobContext context) { + VcardParser(Case currentCase, IngestJobContext context, Map customAttributeCache) { this.context = context; this.currentCase = currentCase; tskCase = currentCase.getSleuthkitCase(); blackboard = tskCase.getBlackboard(); fileManager = currentCase.getServices().getFileManager(); + this.customAttributeCache = customAttributeCache; } /** @@ -421,26 +427,29 @@ final class VcardParser { if (splitType != null && !splitType.isEmpty()) { attributeTypeName = "TSK_PHONE_NUMBER_" + splitType; } + + final String finalAttrTypeName = attributeTypeName; - try { - BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName); - if (attributeType == null) { - try{ - // Add this attribute type to the case database. - attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName, - BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, - String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase()))); - - ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); - }catch (BlackboardException ex) { - logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } - } - - } catch (TskCoreException ex) { - logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + // handled in computeIfAbsent to remove concurrency issues when adding to this concurrent hashmap. + BlackboardAttribute.Type attributeType + = this.customAttributeCache.computeIfAbsent(finalAttrTypeName, k -> { + try { + // Add this attribute type to the case database. + return tskCase.getBlackboard().getOrAddAttributeType(finalAttrTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + + } catch (BlackboardException ex) { + VcardParser.logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", + finalAttrTypeName, abstractFile.getName(), abstractFile.getId()), ex); + return null; + } + }); + + if (attributeType != null) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes); } - } + } } } @@ -469,30 +478,37 @@ final class VcardParser { * ez-vcard. Therefore, we must read them manually * ourselves. */ - List splitEmailTypes = Arrays.asList( - type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); + List splitEmailTypes = Arrays.asList( + type.getValue().toUpperCase().replaceAll("\\s+", "").split(",")); - if (splitEmailTypes.size() > 0) { - String splitType = splitEmailTypes.get(0); - String attributeTypeName = "TSK_EMAIL_" + splitType; - if(splitType.isEmpty()) { - attributeTypeName = "TSK_EMAIL"; - } - try { - BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName); - if (attributeType == null) { - // Add this attribute type to the case database. - attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName, - BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, - String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); - } - ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } catch (BlackboardException ex) { - logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); - } - } + if (splitEmailTypes.size() > 0) { + String splitType = splitEmailTypes.get(0); + String attributeTypeName = "TSK_EMAIL_" + splitType; + if (splitType.isEmpty()) { + attributeTypeName = "TSK_EMAIL"; + } + + final String finalAttributeTypeName = attributeTypeName; + + BlackboardAttribute.Type attributeType + = this.customAttributeCache.computeIfAbsent(finalAttributeTypeName, k -> { + try { + // Add this attribute type to the case database. + return tskCase.getBlackboard().getOrAddAttributeType(finalAttributeTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + } catch (BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", + finalAttributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } + + return null; + }); + + if (attributeType != null) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); + } + } } } From 7a7a48762f74e17db33b205d45ebe8cae45877ee Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 25 Jan 2022 10:27:09 -0500 Subject: [PATCH 04/12] bug fixes --- .../ThunderbirdMboxFileIngestModule.java | 22 ++++++++++++++----- .../thunderbirdparser/VcardParser.java | 5 +++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 0000847c63..13ca786c3b 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -84,9 +85,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; - // A cache of custom attributes for the VcardParser unique to each ingest run. - private Map customAttributeCache; - + // A cache of custom attributes for the VcardParser unique to each ingest run, but consistent across threads. + private static ConcurrentMap customAttributeCache = new ConcurrentHashMap<>(); + private static Object customAttributeCacheLock = new Object(); + private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; @@ -100,7 +102,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; - this.customAttributeCache = new ConcurrentHashMap<>(); + + synchronized(customAttributeCacheLock) { + if (!customAttributeCache.isEmpty()) { + customAttributeCache.clear(); + } + } + try { currentCase = Case.getCurrentCaseThrows(); fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); @@ -917,7 +925,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { @Override public void shutDown() { - // nothing to shut down + synchronized(customAttributeCacheLock) { + if (!customAttributeCache.isEmpty()) { + customAttributeCache.clear(); + } + } } } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index 6e3ff0082b..306843f025 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -40,6 +40,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentMap; import java.util.logging.Level; import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; @@ -102,12 +103,12 @@ final class VcardParser { * A custom attribute cache provided to every VcardParser from the * ThunderbirdMboxFileIngestModule, but unique to one ingest run. */ - private final Map customAttributeCache; + private final ConcurrentMap customAttributeCache; /** * Create a VcardParser object. */ - VcardParser(Case currentCase, IngestJobContext context, Map customAttributeCache) { + VcardParser(Case currentCase, IngestJobContext context, ConcurrentMap customAttributeCache) { this.context = context; this.currentCase = currentCase; tskCase = currentCase.getSleuthkitCase(); From 566f1ca4eec28521790b3b11e00502b90d0d711b Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 1 Feb 2022 16:33:05 -0500 Subject: [PATCH 05/12] Only posting ingest inbox messages for the first 20 hits for each KW List term --- .../autopsy/keywordsearch/KeywordList.java | 2 +- .../autopsy/keywordsearch/QueryResults.java | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java index f2db250baa..ea2b150d7f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java @@ -165,7 +165,7 @@ public class KeywordList { /** * Gets the keywords included in the list * - * @return A colleciton of Keyword objects. + * @return A collection of Keyword objects. */ public List getKeywords() { return keywords; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 5ac61eb946..e48375ed0b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -56,6 +56,8 @@ class QueryResults { private static final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName(); private final KeywordSearchQuery query; private final Map> results = new HashMap<>(); + + private static final int MAX_INBOX_NOTIFICATIONS_PER_KW_TERM = 20; /** * Constructs a object that stores and processes the results of a keyword @@ -142,6 +144,14 @@ class QueryResults { */ void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { final Collection hitArtifacts = new ArrayList<>(); + + /* + * Reduce the hits for this keyword to one hit per text source object so + * that only one hit artifact is generated per text source object, no + * matter how many times the keyword was actually found. + */ + int notificationCount = 0; + for (final Keyword keyword : getKeywords()) { /* * Cancellation check. @@ -151,11 +161,6 @@ class QueryResults { return; } - /* - * Reduce the hits for this keyword to one hit per text source - * object so that only one hit artifact is generated per text source - * object, no matter how many times the keyword was actually found. - */ for (KeywordHit hit : getOneHitPerTextSourceObject(keyword)) { /* * Get a snippet (preview) for the hit. Regex queries always @@ -200,8 +205,12 @@ class QueryResults { */ if (null != artifact) { hitArtifacts.add(artifact); - if (notifyInbox) { + if (notifyInbox && notificationCount < MAX_INBOX_NOTIFICATIONS_PER_KW_TERM) { + // only send ingest inbox messages for the first MAX_INBOX_NOTIFICATIONS_PER_KW_TERM hits + // for every KW term (per ingest job, aka data source). Otherwise we can have a situation + // where we send tens of thousands of notifications. try { + notificationCount++; writeSingleFileInboxMessage(artifact, content); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error sending message to ingest messages inbox", ex); //NON-NLS From 330d74b25e6cbe8fd63ea8e05ce5a2388c2f4009 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 4 Feb 2022 10:00:14 -0500 Subject: [PATCH 06/12] Minor --- .../autopsy/keywordsearch/QueryResults.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index e48375ed0b..1c6bfeba1f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -145,13 +145,7 @@ class QueryResults { void process(SwingWorker worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) { final Collection hitArtifacts = new ArrayList<>(); - /* - * Reduce the hits for this keyword to one hit per text source object so - * that only one hit artifact is generated per text source object, no - * matter how many times the keyword was actually found. - */ int notificationCount = 0; - for (final Keyword keyword : getKeywords()) { /* * Cancellation check. @@ -160,7 +154,12 @@ class QueryResults { logger.log(Level.INFO, "Processing cancelled, exiting before processing search term {0}", keyword.getSearchTerm()); //NON-NLS return; } - + + /* + * Reduce the hits for this keyword to one hit per text source + * object so that only one hit artifact is generated per text source + * object, no matter how many times the keyword was actually found. + */ for (KeywordHit hit : getOneHitPerTextSourceObject(keyword)) { /* * Get a snippet (preview) for the hit. Regex queries always From 909313947f1922a7c0e20cef2217d6a9c2b0c8cd Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 7 Feb 2022 11:21:08 -0500 Subject: [PATCH 07/12] hold on to a reference to the extract action helper --- .../src/org/sleuthkit/autopsy/directorytree/ExtractAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java index 0efe3bdcfb..6a585d724f 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java @@ -43,6 +43,8 @@ public final class ExtractAction extends AbstractAction { } return instance; } + + private final ExtractActionHelper extractor = new ExtractActionHelper(); /** * Private constructor for the action. @@ -61,7 +63,6 @@ public final class ExtractAction extends AbstractAction { public void actionPerformed(ActionEvent e) { Lookup lookup = Utilities.actionsGlobalContext(); Collection selectedFiles =lookup.lookupAll(AbstractFile.class); - ExtractActionHelper extractor = new ExtractActionHelper(); extractor.extract(e, selectedFiles); } From c69c058c1858b7c0c663ef7df566c52aa63405db Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Mon, 7 Feb 2022 11:59:50 -0500 Subject: [PATCH 08/12] overwrite confirm --- .../actionhelpers/ExtractActionHelper.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java index 5a28bd9c12..5181c005dd 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java @@ -36,6 +36,7 @@ import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; +import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.FileUtil; @@ -68,12 +69,7 @@ public class ExtractActionHelper { if (selectedFiles.size() > 1) { extractFiles(event, selectedFiles); } else if (selectedFiles.size() == 1) { - AbstractFile source = selectedFiles.iterator().next(); - if (source.isDir()) { - extractFiles(event, selectedFiles); - } else { - extractFile(event, selectedFiles.iterator().next()); - } + extractFile(event, selectedFiles.iterator().next()); } } @@ -83,7 +79,11 @@ public class ExtractActionHelper { * @param event * @param selectedFile Selected file */ - @NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available."}) + @NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available.", + "ExtractActionHelper.extractOverwrite.title=Export to csv file", + "# {0} - fileName", + "ExtractActionHelper.extractOverwrite.msg=A file already exists at {0}. Do you want to overwrite the existing file?" + }) private void extractFile(ActionEvent event, AbstractFile selectedFile) { Case openCase; try { @@ -98,7 +98,19 @@ public class ExtractActionHelper { // If there is an attribute name, change the ":". Otherwise the extracted file will be hidden fileChooser.setSelectedFile(new File(FileUtil.escapeFileName(selectedFile.getName()))); if (fileChooser.showSaveDialog((Component) event.getSource()) == JFileChooser.APPROVE_OPTION) { - updateExportDirectory(fileChooser.getSelectedFile().getParent(), openCase); + File saveLocation = fileChooser.getSelectedFile(); + if (saveLocation.exists()) { + if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), + Bundle.ExtractActionHelper_extractOverwrite_msg(saveLocation.getPath()), + Bundle.ExtractActionHelper_extractOverwrite_title(), + JOptionPane.YES_NO_OPTION)) { + } else { + return; + } + } + + String exportDirectory = saveLocation.getParent(); + updateExportDirectory(exportDirectory, openCase); ArrayList fileExtractionTasks = new ArrayList<>(); fileExtractionTasks.add(new FileExtractionTask(selectedFile, fileChooser.getSelectedFile())); From 3af245e41186b024a4f531dcda3ee65c42dbb8d0 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Feb 2022 09:05:44 -0500 Subject: [PATCH 09/12] don't overwrite on conflict --- .../autopsy/datamodel/ContentUtils.java | 90 ++++++++----------- .../actionhelpers/ExtractActionHelper.java | 84 ++++++++++++++--- 2 files changed, 108 insertions(+), 66 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index 6def78b83d..c73ccb80a4 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -392,85 +392,69 @@ public final class ContentUtils { public static void extract(Content cntnt, java.io.File dest, ProgressHandle progress, SwingWorker worker) { cntnt.accept(new ExtractFscContentVisitor<>(dest, progress, worker, true)); } + + /** + * Base method writing a file to disk. + * + * @param file The TSK content file. + * @param dest The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @param source true if source file + * + * @throws IOException + */ + protected void writeFile(Content file, java.io.File dest, ProgressHandle progress, SwingWorker worker, boolean source) throws IOException { + ContentUtils.writeToFile(file, dest, progress, worker, source); + } - @Override - public Void visit(File file) { + /** + * Visits a TSK content file and writes that file to disk. + * @param file The file to be written. + * @param fileType The file type (i.e. "derived file") for error logging. + * @return null. + */ + protected Void visitFile(Content file, String fileType) { try { - ContentUtils.writeToFile(file, dest, progress, worker, source); + writeFile(file, dest, progress, worker, source); } catch (ReadContentInputStreamException ex) { logger.log(Level.WARNING, String.format("Error reading file '%s' (id=%d).", file.getName(), file.getId()), ex); //NON-NLS } catch (IOException ex) { logger.log(Level.SEVERE, - String.format("Error extracting file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS + String.format("Error extracting %s '%s' (id=%d) to '%s'.", + fileType, file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS } return null; } + + @Override + public Void visit(File file) { + return visitFile(file, "file"); + } @Override public Void visit(LayoutFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting unallocated content file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "unallocated content file"); } @Override public Void visit(DerivedFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting derived file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "derived file"); } @Override public Void visit(LocalFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting local file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "local file"); } @Override public Void visit(SlackFile file) { - try { - ContentUtils.writeToFile(file, dest, progress, worker, source); - } catch (ReadContentInputStreamException ex) { - logger.log(Level.WARNING, - String.format("Error reading file '%s' (id=%d).", - file.getName(), file.getId()), ex); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, - String.format("Error extracting slack file '%s' (id=%d) to '%s'.", - file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS - } - return null; + return visitFile(file, "slack file"); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java index 5181c005dd..48714e31e9 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java @@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.directorytree.actionhelpers; import java.awt.Component; import java.awt.event.ActionEvent; import java.io.File; +import java.io.IOException; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -36,15 +38,16 @@ import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; -import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.datamodel.ContentUtils.ExtractFscContentVisitor; import org.sleuthkit.autopsy.guiutils.JFileChooserFactory; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; /** * Helper class for methods needed by actions which extract files. @@ -53,7 +56,7 @@ public class ExtractActionHelper { private final Logger logger = Logger.getLogger(ExtractActionHelper.class.getName()); private String userDefinedExportPath; - + private final JFileChooserFactory extractFileHelper = new JFileChooserFactory(); private final JFileChooserFactory extractFilesHelper = new JFileChooserFactory(); @@ -99,15 +102,15 @@ public class ExtractActionHelper { fileChooser.setSelectedFile(new File(FileUtil.escapeFileName(selectedFile.getName()))); if (fileChooser.showSaveDialog((Component) event.getSource()) == JFileChooser.APPROVE_OPTION) { File saveLocation = fileChooser.getSelectedFile(); - if (saveLocation.exists()) { - if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), - Bundle.ExtractActionHelper_extractOverwrite_msg(saveLocation.getPath()), - Bundle.ExtractActionHelper_extractOverwrite_title(), - JOptionPane.YES_NO_OPTION)) { - } else { - return; - } - } +// if (saveLocation.exists()) { +// if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), +// Bundle.ExtractActionHelper_extractOverwrite_msg(saveLocation.getPath()), +// Bundle.ExtractActionHelper_extractOverwrite_title(), +// JOptionPane.YES_NO_OPTION)) { +// } else { +// return; +// } +// } String exportDirectory = saveLocation.getParent(); updateExportDirectory(exportDirectory, openCase); @@ -281,6 +284,62 @@ public class ExtractActionHelper { } } + /** + * A file content extraction visitor that handles for the UI designed to + * handle file name conflicts by appending the object id to the file name. + */ + private static class UIExtractionVisitor extends ExtractFscContentVisitor { + + /** + * @param file The TSK content file. + * @param dest The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @param source true if source file + */ + UIExtractionVisitor(File dest, ProgressHandle progress, SwingWorker worker, boolean source) { + super(dest, progress, worker, source); + } + + /** + * Writes content and children to disk. + * + * @param content The root content. + * @param file The TSK content file. + * @param dest The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @param source true if source file + */ + static void writeContent(Content content, File dest, ProgressHandle progress, SwingWorker worker) { + content.accept(new UIExtractionVisitor<>(dest, progress, worker, true)); + } + + + @Override + protected void writeFile(Content file, File dest, ProgressHandle progress, SwingWorker worker, boolean source) throws IOException { + File destFile; + if (dest.exists()) { + String parent = dest.getParent(); + String fileName = dest.getName(); + String objIdFileName = MessageFormat.format("{0}-{1}", file.getId(), fileName); + destFile = new File(parent, objIdFileName); + } else { + destFile = dest; + } + + super.writeFile(file, destFile, progress, worker, source); + } + + + } + /** * Thread that does the actual extraction work */ @@ -333,8 +392,7 @@ public class ExtractActionHelper { // Do the extraction tasks. for (FileExtractionTask task : this.extractionTasks) { progress.progress(Bundle.ExtractActionHelper_progress_fileExtracting(task.destination.getName())); - - ContentUtils.ExtractFscContentVisitor.extract(task.source, task.destination, null, this); + UIExtractionVisitor.writeContent(task.source, task.destination, null, this); } return null; From 06f7c5456413df717e400bb6698ca4acece8bdba Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Feb 2022 09:09:25 -0500 Subject: [PATCH 10/12] remove commented out code --- .../actionhelpers/ExtractActionHelper.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java index 48714e31e9..f42b5f1a62 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java @@ -101,19 +101,7 @@ public class ExtractActionHelper { // If there is an attribute name, change the ":". Otherwise the extracted file will be hidden fileChooser.setSelectedFile(new File(FileUtil.escapeFileName(selectedFile.getName()))); if (fileChooser.showSaveDialog((Component) event.getSource()) == JFileChooser.APPROVE_OPTION) { - File saveLocation = fileChooser.getSelectedFile(); -// if (saveLocation.exists()) { -// if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), -// Bundle.ExtractActionHelper_extractOverwrite_msg(saveLocation.getPath()), -// Bundle.ExtractActionHelper_extractOverwrite_title(), -// JOptionPane.YES_NO_OPTION)) { -// } else { -// return; -// } -// } - - String exportDirectory = saveLocation.getParent(); - updateExportDirectory(exportDirectory, openCase); + updateExportDirectory(fileChooser.getSelectedFile().getParent(), openCase); ArrayList fileExtractionTasks = new ArrayList<>(); fileExtractionTasks.add(new FileExtractionTask(selectedFile, fileChooser.getSelectedFile())); From 87d7a40c42e50824bfd2d1ffbda44a0157c5dfda Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 8 Feb 2022 09:40:18 -0500 Subject: [PATCH 11/12] child visitor fix --- .../autopsy/datamodel/ContentUtils.java | 17 +++++++++++++++-- .../actionhelpers/ExtractActionHelper.java | 6 ++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java index c73ccb80a4..0766316e67 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java @@ -477,6 +477,20 @@ public final class ContentUtils { + content.getName(); return new java.io.File(path); } + + /** + * Returns a visitor to visit any child content. + * @param childFile The disk location where the content will be written. + * @param progress progress bar handle to update, if available. null + * otherwise + * @param worker the swing worker background thread the process runs + * within, or null, if in the main thread, used to + * handle task cancellation + * @return + */ + protected ExtractFscContentVisitor getChildVisitor(java.io.File childFile, ProgressHandle progress, SwingWorker worker) { + return new ExtractFscContentVisitor<>(childFile, progress, worker, false); + } public Void visitDir(AbstractFile dir) { @@ -493,8 +507,7 @@ public final class ContentUtils { for (Content child : dir.getChildren()) { if (child instanceof AbstractFile) { //ensure the directory's artifact children are ignored java.io.File childFile = getFsContentDest(child); - ExtractFscContentVisitor childVisitor - = new ExtractFscContentVisitor<>(childFile, progress, worker, false); + ExtractFscContentVisitor childVisitor = getChildVisitor(childFile, progress, worker); // If this is the source directory of an extract it // will have a progress and worker, and will keep track // of the progress bar's progress diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java index f42b5f1a62..386240ea86 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java @@ -325,6 +325,12 @@ public class ExtractActionHelper { super.writeFile(file, destFile, progress, worker, source); } + @Override + protected ExtractFscContentVisitor getChildVisitor(File childFile, ProgressHandle progress, SwingWorker worker) { + return new UIExtractionVisitor(childFile, progress, worker, false); + } + + } From 4ea2f1fb09021df5bde628678c26fc077882a3f9 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 11 Feb 2022 13:27:58 -0500 Subject: [PATCH 12/12] make action synchronous to avoid race condition --- .../autopsy/ingest/IngestJobExecutor.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java index 44b28c415a..f665c4bcf3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; @@ -950,18 +951,25 @@ final class IngestJobExecutor { currentDataSourceIngestModuleCancelled = false; cancelledDataSourceIngestModules.add(moduleDisplayName); if (usingNetBeansGUI && !jobCancelled) { - SwingUtilities.invokeLater(() -> { - /** - * A new progress bar must be created because the cancel button - * of the previously constructed component is disabled by - * NetBeans when the user selects the "OK" button of the - * cancellation confirmation dialog popped up by NetBeans when - * the progress bar cancel button is pressed. - */ - dataSourceIngestProgressBar.finish(); - dataSourceIngestProgressBar = null; - startDataSourceIngestProgressBar(); - }); + try { + // use invokeAndWait to ensure synchronous behavior. + // See JIRA-8298 for more information. + SwingUtilities.invokeAndWait(() -> { + /** + * A new progress bar must be created because the cancel + * button of the previously constructed component is + * disabled by NetBeans when the user selects the "OK" + * button of the cancellation confirmation dialog popped up + * by NetBeans when the progress bar cancel button is + * pressed. + */ + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + startDataSourceIngestProgressBar(); + }); + } catch (InvocationTargetException | InterruptedException ex) { + logger.log(Level.WARNING, "Cancellation worker cancelled.", ex); + } } }