mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7554 from eugene7646/merge_develop

Browse Source 
Merge develop
This commit is contained in:
eugene7646 2022-02-14 12:36:44 -05:00 committed by GitHub
commit e3f663bcc3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 293 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
Core/src/org/sleuthkit/autopsy/datamodel/ContentUtils.java
View File

@ -393,84 +393,68 @@ public final class ContentUtils {
cntnt.accept(new ExtractFscContentVisitor<>(dest, progress, worker, true));
}
@Override
public Void visit(File file) {
try {
/**
* Base method writing a file to disk.
*
* @param file The TSK content file.
* @param dest The disk location where the content will be written.
* @param progress progress bar handle to update, if available. null
* otherwise
* @param worker the swing worker background thread the process runs
* within, or null, if in the main thread, used to
* handle task cancellation
* @param source true if source file
*
* @throws IOException
*/
protected void writeFile(Content file, java.io.File dest, ProgressHandle progress, SwingWorker<T, V> worker, boolean source) throws IOException {
ContentUtils.writeToFile(file, dest, progress, worker, source);
}
/**
* Visits a TSK content file and writes that file to disk.
* @param file The file to be written.
* @param fileType The file type (i.e. "derived file") for error logging.
* @return null.
*/
protected Void visitFile(Content file, String fileType) {
try {
writeFile(file, dest, progress, worker, source);
} catch (ReadContentInputStreamException ex) {
logger.log(Level.WARNING,
String.format("Error reading file '%s' (id=%d).",
file.getName(), file.getId()), ex); //NON-NLS
} catch (IOException ex) {
logger.log(Level.SEVERE,
String.format("Error extracting file '%s' (id=%d) to '%s'.",
file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
String.format("Error extracting %s '%s' (id=%d) to '%s'.",
fileType, file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
}
return null;
}
@Override
public Void visit(File file) {
return visitFile(file, "file");
}
@Override
public Void visit(LayoutFile file) {
try {
ContentUtils.writeToFile(file, dest, progress, worker, source);
} catch (ReadContentInputStreamException ex) {
logger.log(Level.WARNING,
String.format("Error reading file '%s' (id=%d).",
file.getName(), file.getId()), ex); //NON-NLS
} catch (IOException ex) {
logger.log(Level.SEVERE,
String.format("Error extracting unallocated content file '%s' (id=%d) to '%s'.",
file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
}
return null;
return visitFile(file, "unallocated content file");
}
@Override
public Void visit(DerivedFile file) {
try {
ContentUtils.writeToFile(file, dest, progress, worker, source);
} catch (ReadContentInputStreamException ex) {
logger.log(Level.WARNING,
String.format("Error reading file '%s' (id=%d).",
file.getName(), file.getId()), ex); //NON-NLS
} catch (IOException ex) {
logger.log(Level.SEVERE,
String.format("Error extracting derived file '%s' (id=%d) to '%s'.",
file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
}
return null;
return visitFile(file, "derived file");
}
@Override
public Void visit(LocalFile file) {
try {
ContentUtils.writeToFile(file, dest, progress, worker, source);
} catch (ReadContentInputStreamException ex) {
logger.log(Level.WARNING,
String.format("Error reading file '%s' (id=%d).",
file.getName(), file.getId()), ex); //NON-NLS
} catch (IOException ex) {
logger.log(Level.SEVERE,
String.format("Error extracting local file '%s' (id=%d) to '%s'.",
file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
}
return null;
return visitFile(file, "local file");
}
@Override
public Void visit(SlackFile file) {
try {
ContentUtils.writeToFile(file, dest, progress, worker, source);
} catch (ReadContentInputStreamException ex) {
logger.log(Level.WARNING,
String.format("Error reading file '%s' (id=%d).",
file.getName(), file.getId()), ex); //NON-NLS
} catch (IOException ex) {
logger.log(Level.SEVERE,
String.format("Error extracting slack file '%s' (id=%d) to '%s'.",
file.getName(), file.getId(), dest.getAbsolutePath()), ex); //NON-NLS
}
return null;
return visitFile(file, "slack file");
}
@Override
@ -494,6 +478,20 @@ public final class ContentUtils {
return new java.io.File(path);
}
/**
* Returns a visitor to visit any child content.
* @param childFile The disk location where the content will be written.
* @param progress progress bar handle to update, if available. null
* otherwise
* @param worker the swing worker background thread the process runs
* within, or null, if in the main thread, used to
* handle task cancellation
* @return
*/
protected ExtractFscContentVisitor<T, V> getChildVisitor(java.io.File childFile, ProgressHandle progress, SwingWorker<T, V> worker) {
return new ExtractFscContentVisitor<>(childFile, progress, worker, false);
}
public Void visitDir(AbstractFile dir) {
// don't extract . and .. directories
@ -509,8 +507,7 @@ public final class ContentUtils {
for (Content child : dir.getChildren()) {
if (child instanceof AbstractFile) { //ensure the directory's artifact children are ignored
java.io.File childFile = getFsContentDest(child);
ExtractFscContentVisitor<T, V> childVisitor
= new ExtractFscContentVisitor<>(childFile, progress, worker, false);
ExtractFscContentVisitor<T, V> childVisitor = getChildVisitor(childFile, progress, worker);
// If this is the source directory of an extract it
// will have a progress and worker, and will keep track
// of the progress bar's progress

3
Core/src/org/sleuthkit/autopsy/directorytree/ExtractAction.java
View File

@ -44,6 +44,8 @@ public final class ExtractAction extends AbstractAction {
return instance;
}
private final ExtractActionHelper extractor = new ExtractActionHelper();
/**
* Private constructor for the action.
*/
@ -61,7 +63,6 @@ public final class ExtractAction extends AbstractAction {
public void actionPerformed(ActionEvent e) {
Lookup lookup = Utilities.actionsGlobalContext();
Collection<? extends AbstractFile> selectedFiles =lookup.lookupAll(AbstractFile.class);
ExtractActionHelper extractor = new ExtractActionHelper();
extractor.extract(e, selectedFiles);
}

80
Core/src/org/sleuthkit/autopsy/directorytree/actionhelpers/ExtractActionHelper.java
View File

@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.directorytree.actionhelpers;
import java.awt.Component;
import java.awt.event.ActionEvent;
import java.io.File;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@ -42,8 +44,10 @@ import org.sleuthkit.autopsy.coreutils.FileUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.datamodel.ContentUtils.ExtractFscContentVisitor;
import org.sleuthkit.autopsy.guiutils.JFileChooserFactory;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
/**
* Helper class for methods needed by actions which extract files.
@ -68,14 +72,9 @@ public class ExtractActionHelper {
if (selectedFiles.size() > 1) {
extractFiles(event, selectedFiles);
} else if (selectedFiles.size() == 1) {
AbstractFile source = selectedFiles.iterator().next();
if (source.isDir()) {
extractFiles(event, selectedFiles);
} else {
extractFile(event, selectedFiles.iterator().next());
}
}
}
/**
* Called when user has selected a single file to extract
@ -83,7 +82,11 @@ public class ExtractActionHelper {
* @param event
* @param selectedFile Selected file
*/
@NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available."})
@NbBundle.Messages({"ExtractActionHelper.noOpenCase.errMsg=No open case available.",
"ExtractActionHelper.extractOverwrite.title=Export to csv file",
"# {0} - fileName",
"ExtractActionHelper.extractOverwrite.msg=A file already exists at {0}. Do you want to overwrite the existing file?"
})
private void extractFile(ActionEvent event, AbstractFile selectedFile) {
Case openCase;
try {
@ -269,6 +272,68 @@ public class ExtractActionHelper {
}
}
/**
* A file content extraction visitor that handles for the UI designed to
* handle file name conflicts by appending the object id to the file name.
*/
private static class UIExtractionVisitor<T, V> extends ExtractFscContentVisitor<T, V> {
/**
* @param file The TSK content file.
* @param dest The disk location where the content will be written.
* @param progress progress bar handle to update, if available. null
* otherwise
* @param worker the swing worker background thread the process runs
* within, or null, if in the main thread, used to
* handle task cancellation
* @param source true if source file
*/
UIExtractionVisitor(File dest, ProgressHandle progress, SwingWorker<T, V> worker, boolean source) {
super(dest, progress, worker, source);
}
/**
* Writes content and children to disk.
*
* @param content The root content.
* @param file The TSK content file.
* @param dest The disk location where the content will be written.
* @param progress progress bar handle to update, if available. null
* otherwise
* @param worker the swing worker background thread the process runs
* within, or null, if in the main thread, used to
* handle task cancellation
* @param source true if source file
*/
static <T,V> void writeContent(Content content, File dest, ProgressHandle progress, SwingWorker<T, V> worker) {
content.accept(new UIExtractionVisitor<>(dest, progress, worker, true));
}
@Override
protected void writeFile(Content file, File dest, ProgressHandle progress, SwingWorker<T, V> worker, boolean source) throws IOException {
File destFile;
if (dest.exists()) {
String parent = dest.getParent();
String fileName = dest.getName();
String objIdFileName = MessageFormat.format("{0}-{1}", file.getId(), fileName);
destFile = new File(parent, objIdFileName);
} else {
destFile = dest;
}
super.writeFile(file, destFile, progress, worker, source);
}
@Override
protected ExtractFscContentVisitor<T, V> getChildVisitor(File childFile, ProgressHandle progress, SwingWorker<T, V> worker) {
return new UIExtractionVisitor(childFile, progress, worker, false);
}
}
/**
* Thread that does the actual extraction work
*/
@ -321,8 +386,7 @@ public class ExtractActionHelper {
// Do the extraction tasks.
for (FileExtractionTask task : this.extractionTasks) {
progress.progress(Bundle.ExtractActionHelper_progress_fileExtracting(task.destination.getName()));
ContentUtils.ExtractFscContentVisitor.extract(task.source, task.destination, null, this);
UIExtractionVisitor.writeContent(task.source, task.destination, null, this);
}
return null;

20
Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
@ -950,18 +951,25 @@ final class IngestJobExecutor {
currentDataSourceIngestModuleCancelled = false;
cancelledDataSourceIngestModules.add(moduleDisplayName);
if (usingNetBeansGUI && !jobCancelled) {
SwingUtilities.invokeLater(() -> {
try {
// use invokeAndWait to ensure synchronous behavior.
// See JIRA-8298 for more information.
SwingUtilities.invokeAndWait(() -> {
/**
* A new progress bar must be created because the cancel button
* of the previously constructed component is disabled by
* NetBeans when the user selects the "OK" button of the
* cancellation confirmation dialog popped up by NetBeans when
* the progress bar cancel button is pressed.
* A new progress bar must be created because the cancel
* button of the previously constructed component is
* disabled by NetBeans when the user selects the "OK"
* button of the cancellation confirmation dialog popped up
* by NetBeans when the progress bar cancel button is
* pressed.
*/
dataSourceIngestProgressBar.finish();
dataSourceIngestProgressBar = null;
startDataSourceIngestProgressBar();
});
} catch (InvocationTargetException | InterruptedException ex) {
logger.log(Level.WARNING, "Cancellation worker cancelled.", ex);
}
}
}

4
Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java
View File

@ -231,7 +231,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule {
return;
}
aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile);
aLeappFileProcessor.processFiles(dataSource, moduleOutputPath, aLeappFile, statusHelper);
}
/**
@ -274,7 +274,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule {
return;
}
aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath);
aLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper);
}
/**

3
Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED
View File

@ -46,9 +46,12 @@ LeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts.
LeappFileProcessor.error.creating.output.dir=Error creating Leapp module output directory.
LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory
LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file.
LeappFileProcessor.findTsv=Finding all Leapp ouput
LeappFileProcessor.has.run=Leapp
LeappFileProcessor.Leapp.cancelled=Leapp run was canceled
LeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact
LeappFileProcessor.running.Leapp=Running Leapp
LeappFileProcessor.starting.Leapp=Starting Leapp
# {0} - fileName
LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}
LeappFileProcessor_cannotParseXml=Cannot Parse XML file.

4
Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java
View File

@ -232,7 +232,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule {
return;
}
iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile);
iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile, statusHelper);
}
/**
@ -274,7 +274,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule {
return;
}
iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath);
iLeappFileProcessor.processFileSystem(dataSource, moduleOutputPath, statusHelper);
}
/**

96
Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java
View File

@ -54,6 +54,7 @@ import org.apache.commons.collections4.MapUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
@ -61,6 +62,7 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
@ -221,13 +223,19 @@ public final class LeappFileProcessor {
"LeappFileProcessor.has.run=Leapp",
"LeappFileProcessor.Leapp.cancelled=Leapp run was canceled",
"LeappFileProcessor.completed=Leapp Processing Completed",
"LeappFileProcessor.findTsv=Finding all Leapp ouput",
"LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"
})
public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile) {
public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile, DataSourceIngestModuleProgress progress) {
try {
if (checkCancelled()) {
return ProcessResult.OK;
}
progress.switchToIndeterminate();
progress.progress(Bundle.LeappFileProcessor_findTsv());
List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
processLeappFiles(LeappTsvOutputFiles, LeappFile);
} catch (IOException | IngestModuleException ex) {
processLeappFiles(LeappTsvOutputFiles, LeappFile, progress);
} catch (IngestModuleException ex) {
logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
return ProcessResult.ERROR;
}
@ -235,11 +243,15 @@ public final class LeappFileProcessor {
return ProcessResult.OK;
}
public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath) {
public ProcessResult processFileSystem(Content dataSource, Path moduleOutputPath, DataSourceIngestModuleProgress progress) {
try {
if (checkCancelled()) {
return ProcessResult.OK;
}
progress.switchToIndeterminate();
progress.progress(Bundle.LeappFileProcessor_findTsv());
List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
processLeappFiles(LeappTsvOutputFiles, dataSource);
processLeappFiles(LeappTsvOutputFiles, dataSource, progress);
} catch (IngestModuleException ex) {
logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
return ProcessResult.ERROR;
@ -275,75 +287,58 @@ public final class LeappFileProcessor {
}
/**
* Process the Leapp files that were found that match the xml mapping file
*
* @param LeappFilesToProcess List of files to process
* @param LeappImageFile Abstract file to create artifact for
*
* @throws FileNotFoundException
* @throws IOException
*/
private void processLeappFiles(List<String> LeappFilesToProcess, AbstractFile LeappImageFile) throws FileNotFoundException, IOException, IngestModuleException {
List<BlackboardArtifact> bbartifacts = new ArrayList<>();
for (String LeappFileName : LeappFilesToProcess) {
String fileName = FilenameUtils.getName(LeappFileName);
File LeappFile = new File(LeappFileName);
if (tsvFileAttributes.containsKey(fileName)) {
BlackboardArtifact.Type artifactType = null;
try {
List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
artifactType = tsvFileArtifacts.get(fileName);
processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, LeappImageFile);
} catch (TskCoreException ex) {
throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", artifactType == null ? "<null>" : artifactType.toString()), ex);
private boolean checkCancelled() {
if (this.context.dataSourceIngestIsCancelled()) {
logger.log(Level.INFO, "Leapp File processing module run was cancelled"); //NON-NLS
return true;
} else {
return false;
}
}
}
if (!bbartifacts.isEmpty()) {
postArtifacts(bbartifacts);
}
}
/**
* Process the Leapp files that were found that match the xml mapping file
*
* @param LeappFilesToProcess List of files to process
* @param LeappFilesToProcess List of files to process.
* @param dataSource The data source.
* @param progress Means of updating progress in UI.
*
* @throws FileNotFoundException
* @throws IOException
*/
private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource) throws IngestModuleException {
List<BlackboardArtifact> bbartifacts = new ArrayList<>();
@Messages({
"# {0} - fileName",
"LeappFileProcessor.tsvProcessed=Processing LEAPP output file: {0}"
})
private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource, DataSourceIngestModuleProgress progress) throws IngestModuleException {
progress.switchToDeterminate(LeappFilesToProcess.size());
for (String LeappFileName : LeappFilesToProcess) {
for (int i = 0; i < LeappFilesToProcess.size(); i++) {
if (checkCancelled()) {
return;
}
String LeappFileName = LeappFilesToProcess.get(i);
String fileName = FilenameUtils.getName(LeappFileName);
progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i);
File LeappFile = new File(LeappFileName);
if (tsvFileAttributes.containsKey(fileName)) {
List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(fileName);
try {
processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource);
processFile(LeappFile, attrList, fileName, artifactType, dataSource);
} catch (TskCoreException | IOException ex) {
logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex);
}
}
}
if (!bbartifacts.isEmpty()) {
postArtifacts(bbartifacts);
}
}
private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName, BlackboardArtifact.Type artifactType,
List<BlackboardArtifact> bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException,
private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName,
BlackboardArtifact.Type artifactType, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException,
TskCoreException {
String trackpointSegmentName = null;
@ -358,6 +353,8 @@ public final class LeappFileProcessor {
return;
}
List<BlackboardArtifact> bbartifacts = new ArrayList<>();
// based on https://stackoverflow.com/questions/56921465/jackson-csv-schema-for-array
try (MappingIterator<List<String>> iterator = new CsvMapper()
.enable(CsvParser.Feature.WRAP_AS_ARRAY)
@ -418,6 +415,9 @@ public final class LeappFileProcessor {
throw new IngestModuleException(Bundle.LeappFileProcessor_cannot_create_message_relationship() + ex.getLocalizedMessage(), ex); //NON-NLS
}
if (!bbartifacts.isEmpty()) {
postArtifacts(bbartifacts);
}
}
@NbBundle.Messages({

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordList.java
View File

@ -165,7 +165,7 @@ public class KeywordList {
/**
* Gets the keywords included in the list
*
* @return A colleciton of Keyword objects.
* @return A collection of Keyword objects.
*/
public List<Keyword> getKeywords() {
return keywords;

10
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java
View File

@ -57,6 +57,8 @@ class QueryResults {
private final KeywordSearchQuery query;
private final Map<Keyword, List<KeywordHit>> results = new HashMap<>();
private static final int MAX_INBOX_NOTIFICATIONS_PER_KW_TERM = 20;
/**
* Constructs a object that stores and processes the results of a keyword
* search query. Processing includes adding keyword hit artifacts to the
@ -142,6 +144,8 @@ class QueryResults {
*/
void process(SwingWorker<?, ?> worker, boolean notifyInbox, boolean saveResults, Long ingestJobId) {
final Collection<BlackboardArtifact> hitArtifacts = new ArrayList<>();
int notificationCount = 0;
for (final Keyword keyword : getKeywords()) {
/*
* Cancellation check.
@ -200,8 +204,12 @@ class QueryResults {
*/
if (null != artifact) {
hitArtifacts.add(artifact);
if (notifyInbox) {
if (notifyInbox && notificationCount < MAX_INBOX_NOTIFICATIONS_PER_KW_TERM) {
// only send ingest inbox messages for the first MAX_INBOX_NOTIFICATIONS_PER_KW_TERM hits
// for every KW term (per ingest job, aka data source). Otherwise we can have a situation
// where we send tens of thousands of notifications.
try {
notificationCount++;
writeSingleFileInboxMessage(artifact, content);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error sending message to ingest messages inbox", ex); //NON-NLS

21
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
View File

@ -30,6 +30,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -83,6 +85,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
private Blackboard blackboard;
private CommunicationArtifactsHelper communicationArtifactsHelper;
// A cache of custom attributes for the VcardParser unique to each ingest run, but consistent across threads.
private static ConcurrentMap<String, BlackboardAttribute.Type> customAttributeCache = new ConcurrentHashMap<>();
private static Object customAttributeCacheLock = new Object();
private static final int MBOX_SIZE_TO_SPLIT = 1048576000;
private Case currentCase;
@ -96,6 +102,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
@Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."})
public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context;
synchronized(customAttributeCacheLock) {
if (!customAttributeCache.isEmpty()) {
customAttributeCache.clear();
}
}
try {
currentCase = Case.getCurrentCaseThrows();
fileManager = Case.getCurrentCaseThrows().getServices().getFileManager();
@ -441,7 +454,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
})
private ProcessResult processVcard(AbstractFile abstractFile) {
try {
VcardParser parser = new VcardParser(currentCase, context);
VcardParser parser = new VcardParser(currentCase, context, customAttributeCache);
parser.parse(abstractFile);
} catch (IOException | NoCurrentCaseException ex) {
logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS
@ -912,7 +925,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
@Override
public void shutDown() {
// nothing to shut down
synchronized(customAttributeCacheLock) {
if (!customAttributeCache.isEmpty()) {
customAttributeCache.clear();
}
}
}
}

53
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java
View File

@ -40,6 +40,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import java.util.logging.Level;
import org.apache.commons.lang3.StringUtils;
import org.openide.util.NbBundle;
@ -98,16 +99,22 @@ final class VcardParser {
private final Blackboard blackboard;
private final Case currentCase;
private final SleuthkitCase tskCase;
/**
* A custom attribute cache provided to every VcardParser from the
* ThunderbirdMboxFileIngestModule, but unique to one ingest run.
*/
private final ConcurrentMap<String, BlackboardAttribute.Type> customAttributeCache;
/**
* Create a VcardParser object.
*/
VcardParser(Case currentCase, IngestJobContext context) {
VcardParser(Case currentCase, IngestJobContext context, ConcurrentMap<String, BlackboardAttribute.Type> customAttributeCache) {
this.context = context;
this.currentCase = currentCase;
tskCase = currentCase.getSleuthkitCase();
blackboard = tskCase.getBlackboard();
fileManager = currentCase.getServices().getFileManager();
this.customAttributeCache = customAttributeCache;
}
/**
@ -422,23 +429,26 @@ final class VcardParser {
attributeTypeName = "TSK_PHONE_NUMBER_" + splitType;
}
try {
BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName);
if (attributeType == null) {
final String finalAttrTypeName = attributeTypeName;
// handled in computeIfAbsent to remove concurrency issues when adding to this concurrent hashmap.
BlackboardAttribute.Type attributeType
= this.customAttributeCache.computeIfAbsent(finalAttrTypeName, k -> {
try {
// Add this attribute type to the case database.
attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName,
return tskCase.getBlackboard().getOrAddAttributeType(finalAttrTypeName,
BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING,
String.format("Phone Number (%s)", StringUtils.capitalize(splitType.toLowerCase())));
ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes);
} catch (BlackboardException ex) {
logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
}
VcardParser.logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).",
finalAttrTypeName, abstractFile.getName(), abstractFile.getId()), ex);
return null;
}
});
} catch (TskCoreException ex) {
logger.log(Level.WARNING, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
if (attributeType != null) {
ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephoneText, attributeType, attributes);
}
}
}
@ -478,19 +488,26 @@ final class VcardParser {
if (splitType.isEmpty()) {
attributeTypeName = "TSK_EMAIL";
}
final String finalAttributeTypeName = attributeTypeName;
BlackboardAttribute.Type attributeType
= this.customAttributeCache.computeIfAbsent(finalAttributeTypeName, k -> {
try {
BlackboardAttribute.Type attributeType = tskCase.getBlackboard().getAttributeType(attributeTypeName);
if (attributeType == null) {
// Add this attribute type to the case database.
attributeType = tskCase.getBlackboard().getOrAddAttributeType(attributeTypeName,
return tskCase.getBlackboard().getOrAddAttributeType(finalAttributeTypeName,
BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING,
String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase())));
}
ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
} catch (BlackboardException ex) {
logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).",
finalAttributeTypeName, abstractFile.getName(), abstractFile.getId()), ex);
}
return null;
});
if (attributeType != null) {
ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes);
}
}
}