mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed show stopper bugs in new ingest framework

Browse Source
This commit is contained in:
Richard Cordovano 2014-03-18 00:22:28 -04:00
parent e99925fb7d
commit 03e2f5fb6c
12 changed files with 55 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java
View File

@ -172,7 +172,7 @@ final class DataSourceIngestTask {
IngestModuleContext context = new IngestModuleContext(task, factory);
try {
module.startUp(context);
modulesByClass.put(module.getClass().getCanonicalName(), module);
modulesByClass.put(module.getClassName(), module);
IngestManager.fireModuleEvent(IngestManager.IngestModuleEvent.STARTED.toString(), factory.getModuleDisplayName());
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
@ -299,7 +299,7 @@ final class DataSourceIngestTask {
IngestModuleContext context = new IngestModuleContext(task, factory);
try {
module.startUp(context);
modulesByClass.put(module.getClass().getCanonicalName(), module);
modulesByClass.put(module.getClassName(), module);
IngestManager.fireModuleEvent(IngestManager.IngestModuleEvent.STARTED.toString(), factory.getModuleDisplayName());
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));

11
Core/src/org/sleuthkit/autopsy/ingest/IngestConfigurator.java
View File

@ -174,15 +174,16 @@ public class IngestConfigurator {
// IngestConfigurator class.
public void start() {
// Filter out the disabled module tremplates.
List<IngestModuleTemplate> enabledModuleTemplates = new ArrayList<>();
List<IngestModuleTemplate> moduleTemplates = ingestConfigPanel.getIngestModuleTemplates();
for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
if (!moduleTemplate.isEnabled()) {
moduleTemplates.remove(moduleTemplate);
if (moduleTemplate.isEnabled()) {
enabledModuleTemplates.add(moduleTemplate);
}
}
if (!moduleTemplates.isEmpty() && null != contentToIngest) {
IngestManager.getDefault().scheduleDataSourceTasks(contentToIngest, moduleTemplates, ingestConfigPanel.getProcessUnallocSpace());
if ((!enabledModuleTemplates.isEmpty()) && (contentToIngest != null)) {
IngestManager.getDefault().scheduleDataSourceTasks(contentToIngest, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace());
}
}
@ -209,7 +210,7 @@ public class IngestConfigurator {
return csvList.toString();
}
// RJCTODO: May need additional mappings
// RJCTODO: May need additional mappings - EWF Verify to EWF Verifier
private HashSet<String> getModulesNamesFromSetting(String key, String defaultSetting) {
// Get the ingest modules setting from the user's config file.
// If there is no such setting yet, create the default setting.

19
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java
View File

@ -26,7 +26,7 @@ import org.openide.util.Lookup;
import org.sleuthkit.autopsy.coreutils.Logger;
/**
* Looks up loaded ingest module factories using NetBean global lookup.
* Looks up loaded ingest module factories using the NetBean global lookup.
*/
final class IngestModuleLoader {
@ -35,7 +35,13 @@ final class IngestModuleLoader {
private final List<IngestModuleFactory> moduleFactories = new ArrayList<>();
private IngestModuleLoader() {
lookUpIngestModuleFactories();
// RJCTODO: Possibly add code to listen to changes in the collection and restore listener code...
// RJCTODO: Need a name uniqueness test/solution?
Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class);
for (IngestModuleFactory factory : factories) {
logger.log(Level.INFO, "Found ingest module factory: name = {0}, version = {1}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()});
moduleFactories.add(factory);
}
}
synchronized static IngestModuleLoader getInstance() {
@ -48,13 +54,4 @@ final class IngestModuleLoader {
List<IngestModuleFactory> getIngestModuleFactories() {
return new ArrayList<>(moduleFactories);
}
private void lookUpIngestModuleFactories() {
// RJCTODO: Possibly add code to listen to changes in the collection and restore listener code...
Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class);
for (IngestModuleFactory factory : factories) {
logger.log(Level.INFO, "Found ingest module factory: name = {0}, version = {1}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()});
moduleFactories.add(factory);
}
}
}

8
Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.ingest;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
@ -71,6 +72,13 @@ final class IngestPipelinesConfiguration {
}
private void readPipelinesConfigurationFile() {
try {
PlatformUtil.extractResourceToUserConfigDir(IngestModuleLoader.class, PIPELINES_CONFIG_FILE);
} catch (IOException ex) {
logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir", ex);
return;
}
String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE;
Document doc = XMLUtil.loadDoc(IngestModuleLoader.class, configFilePath, PIPELINES_CONFIG_FILE_XSD);
if (doc == null) {

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -75,7 +75,7 @@ import org.sleuthkit.datamodel.TskData.FileKnown;
* on currently configured lists for ingest and writes results to blackboard
* Reports interesting events to Inbox and to viewers
*
* Registered as a module in layer.xml
* Registered as a module in layer.xml RJCTODO: Track this down, does not seem to be true
*/
public final class KeywordSearchIngestModule extends IngestModuleAdapter implements FileIngestModule {

11
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java
View File

@ -36,6 +36,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.autopsy.ingest.IngestModule.ResultCode;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.IngestModuleContext;
/**
* Recent activity image ingest module
@ -139,8 +140,7 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da
}
@Override
public void startUp(org.sleuthkit.autopsy.ingest.IngestModuleContext context) Exception {
super.startUp(context);
public void startUp(IngestModuleContext context) throws Exception {
services = IngestServices.getDefault();
Extract registry = new ExtractRegistry();
@ -162,12 +162,7 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da
browserExtracters.add(iexplore);
for (Extract extracter : extracters) {
try {
extracter.init();
} catch (Exception ex) {
logger.log(Level.SEVERE, "Exception during init() of " + extracter.getName(), ex);
throw new IngestModuleException(ex.getMessage());
}
extracter.init();
}
}

12
ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java
View File

@ -26,13 +26,9 @@ import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile;
import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile.ProcessResult;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.IngestModuleContext;
import org.sleuthkit.autopsy.ingest.IngestModuleInit;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleContentEvent;
import org.sleuthkit.autopsy.scalpel.jni.ScalpelOutputParser.CarvedFileMeta;
@ -52,7 +48,6 @@ import org.sleuthkit.datamodel.Volume;
/**
* Scalpel carving ingest module
*/
class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileIngestModule {
private static final Logger logger = Logger.getLogger(ScalpelCarverIngestModule.class.getName());
@ -188,16 +183,13 @@ class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileInges
}
// reschedule carved files
for (LayoutFile carvedFile : carvedFiles) {
is.scheduleFile(carvedFile, pipelineContext);
}
context.submitFilesForIngest(new ArrayList<AbstractFile>(carvedFiles));
return ResultCode.OK;
}
@Override
public void startUp(IngestModuleContext context) throws IngestModuleException {
public void startUp(IngestModuleContext context) throws Exception {
this.context = context;
// make sure this is Windows

2
SevenZip/src/org/sleuthkit/autopsy/sevenzip/ArchiveFileExtractorModuleFactory.java
View File

@ -29,7 +29,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleSettings;
* A factory for creating archive extractor file ingest modules and the user
* interface panels used to configure the settings for instances of the modules.
*/
@ServiceProvider(service = IngestModuleFactory.class)
@ServiceProvider(service=IngestModuleFactory.class)
public class ArchiveFileExtractorModuleFactory extends IngestModuleFactoryAdapter {
static String getModuleName() {

33
SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipIngestModule.java
View File

@ -85,16 +85,17 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
private static final int readHeaderSize = 4;
private final byte[] fileHeaderBuffer = new byte[readHeaderSize];
private static final int ZIP_SIGNATURE_BE = 0x504B0304;
private IngestModuleContext context;
SevenZipIngestModule() {
}
@Override
public void startUp(IngestModuleContext context) throws IngestModuleException{
super.startUp(context);
unpackDir = getContext().getOutputDirectoryRelativePath();
unpackDirPath = getContext().getOutputDirectoryAbsolutePath();
fileManager = getContext().getCase().getServices().getFileManager();
public void startUp(IngestModuleContext context) throws Exception{
this.context = context;
unpackDir = context.getOutputDirectoryRelativePath();
unpackDirPath = context.getOutputDirectoryAbsolutePath();
fileManager = context.getCase().getServices().getFileManager();
File unpackDirPathFile = new File(unpackDirPath);
if (!unpackDirPathFile.exists()) {
@ -104,7 +105,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
logger.log(Level.SEVERE, "Error initializing output dir: " + unpackDirPath, e);
String msg = "Error initializing archive extractor";
String details = "Error initializing output dir: " + unpackDirPath + ": " + e.getMessage();
getContext().postErrorIngestMessage(++messageID, msg, details);
context.postErrorIngestMessage(++messageID, msg, details);
throw e;
}
}
@ -117,7 +118,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e);
String msg = "Error initializing archive extractor";
String details = "Could not initialize 7-ZIP library: " + e.getMessage();
getContext().postErrorIngestMessage(++messageID, msg, details);
context.postErrorIngestMessage(++messageID, msg, details);
throw new RuntimeException(e);
}
@ -159,7 +160,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
List<AbstractFile> unpackedFiles = unpack(abstractFile);
if (!unpackedFiles.isEmpty()) {
sendNewFilesEvent(abstractFile, unpackedFiles);
getContext().submitFilesForIngest(unpackedFiles);
context.submitFilesForIngest(unpackedFiles);
}
return ResultCode.OK;
@ -226,7 +227,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
+ ", item: " + itemName;
String details = "The archive item compression ratio is " + cRatio
+ ", skipping processing of this archive item. ";
getContext().postWarningIngestMessage(++messageID, msg, details);
context.postWarningIngestMessage(++messageID, msg, details);
return true;
} else {
return false;
@ -257,7 +258,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String msg = "Possible ZIP bomb detected: " + archiveFile.getName();
String details = "The archive is " + parentAr.getDepth()
+ " levels deep, skipping processing of this archive and its contents ";
getContext().postWarningIngestMessage(++messageID, msg, details);
context.postWarningIngestMessage(++messageID, msg, details);
return unpackedFiles;
}
@ -377,7 +378,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
if (newDiskSpace < MIN_FREE_DISK_SPACE) {
String msg = "Not enough disk space to unpack archive item: " + archiveFile.getName() + ", " + fileName;
String details = "The archive item is too large to unpack, skipping unpacking this item. ";
getContext().postErrorIngestMessage(++messageID, msg, details);
context.postErrorIngestMessage(++messageID, msg, details);
logger.log(Level.INFO, "Skipping archive item due not sufficient disk space for this item: {0}, {1}", new Object[]{archiveFile.getName(), fileName});
continue; //skip this file
} else {
@ -472,7 +473,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String details = "Error unpacking ("
+ (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC) ? "allocated" : "deleted") + ") " + fullName
+ ". " + ex.getMessage();
getContext().postErrorIngestMessage(++messageID, msg, details);
context.postErrorIngestMessage(++messageID, msg, details);
} finally {
if (inArchive != null) {
try {
@ -499,15 +500,15 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
try {
BlackboardArtifact artifact = archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), getContext().getModuleDisplayName(), encryptionType));
getContext().fireDataEvent(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), ArchiveFileExtractorModuleFactory.getModuleName(), encryptionType));
context.fireDataEvent(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error creating blackboard artifact for encryption detected for file: " + archiveFile, ex);
}
String msg = "Encrypted files in archive detected. ";
String details = "Some files in archive: " + archiveFile.getName() + " are encrypted. Archive extractor was unable to extract all files from this archive.";
getContext().postWarningIngestMessage(++messageID, msg, details);
context.postWarningIngestMessage(++messageID, msg, details);
}
return unpackedFiles;
@ -715,7 +716,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
try {
DerivedFile df = fileManager.addDerivedFile(fileName, localRelPath, size,
node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
isFile, parent, "", getContext().getModuleDisplayName(), "", "");
isFile, parent, "", ArchiveFileExtractorModuleFactory.getModuleName(), "", "");
node.setFile(df);

2
ewfVerify/src/org/sleuthkit/autopsy/ewfverify/EwfVerifierModuleFactory.java
View File

@ -37,7 +37,7 @@ public class EwfVerifierModuleFactory extends IngestModuleFactoryAdapter {
}
static String getModuleName() {
return "EWF Verify"; // RJCTODO: Is this what we want here?
return "EWF Verify"; // RJCTODO: Is this what we want here? Also, this class is not in pipeline config
}
@Override

2
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailParserModuleFactory.java
View File

@ -29,7 +29,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleSettings;
* A factory for creating email parser file ingest modules and the user
* interface panels used to configure the settings for instances of the modules.
*/
@ServiceProvider(service = IngestModuleFactory.class)
@ServiceProvider(service=IngestModuleFactory.class)
public class EmailParserModuleFactory extends IngestModuleFactoryAdapter {
static String getModuleName() {

19
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
View File

@ -53,8 +53,9 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
private IngestServices services;
private final String hashDBModuleName = "Hash Lookup";
private int messageId = 0;
private int messageId = 0; // RJCTODO: Not thread safe
private FileManager fileManager;
private IngestModuleContext context;
ThunderbirdMboxFileIngestModule() {
}
@ -72,16 +73,6 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
return ResultCode.OK;
}
// RJCTODO: We may be able to jettision this now
//file has read error, stop processing it
// @@@ I don't really like this
// we don't know if Hash was run or if it had lookup errors
// IngestModuleAbstractFile.ResultCode hashDBResult =
// services.getAbstractFileModuleResult(hashDBModuleName);
// if (hashDBResult == IngestModuleAbstractFile.ResultCode.ERROR) {
// return ResultCode.ERROR;
// }
if (abstractFile.isVirtual()) {
return ResultCode.OK;
}
@ -257,8 +248,8 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
}
@Override
public void startUp(IngestModuleContext context) {
setContext(context);
public void startUp(IngestModuleContext context) throws Exception {
this.context = context;
services = IngestServices.getDefault();
fileManager = Case.getCurrentCase().getServices().getFileManager();
}
@ -284,7 +275,7 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
services.fireModuleContentEvent(new ModuleContentEvent(derived));
}
}
getContext().submitFilesForIngest(derivedFiles);
context.submitFilesForIngest(derivedFiles);
services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
}