Fixed show stopper bugs in new ingest framework

Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java

@@ -172,7 +172,7 @@ final class DataSourceIngestTask {
                     IngestModuleContext context = new IngestModuleContext(task, factory);
                     try {
                         module.startUp(context);
-                        modulesByClass.put(module.getClass().getCanonicalName(), module);
+                        modulesByClass.put(module.getClassName(), module);
                         IngestManager.fireModuleEvent(IngestManager.IngestModuleEvent.STARTED.toString(), factory.getModuleDisplayName());
                     } catch (Exception ex) {
                         errors.add(new IngestModuleError(module.getDisplayName(), ex));
@@ -299,7 +299,7 @@ final class DataSourceIngestTask {
                     IngestModuleContext context = new IngestModuleContext(task, factory);
                     try {
                         module.startUp(context);
-                        modulesByClass.put(module.getClass().getCanonicalName(), module);
+                        modulesByClass.put(module.getClassName(), module);
                         IngestManager.fireModuleEvent(IngestManager.IngestModuleEvent.STARTED.toString(), factory.getModuleDisplayName());
                     } catch (Exception ex) {
                         errors.add(new IngestModuleError(module.getDisplayName(), ex));

Core/src/org/sleuthkit/autopsy/ingest/IngestConfigurator.java

@@ -174,15 +174,16 @@ public class IngestConfigurator {
     // IngestConfigurator class.
     public void start() {
         // Filter out the disabled module tremplates.
+        List<IngestModuleTemplate> enabledModuleTemplates = new ArrayList<>();
         List<IngestModuleTemplate> moduleTemplates = ingestConfigPanel.getIngestModuleTemplates();
         for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
-            if (!moduleTemplate.isEnabled()) {
+            if (moduleTemplate.isEnabled()) {
-                moduleTemplates.remove(moduleTemplate);
+                enabledModuleTemplates.add(moduleTemplate);
             }
         }
 
-        if (!moduleTemplates.isEmpty() && null != contentToIngest) {
+        if ((!enabledModuleTemplates.isEmpty()) && (contentToIngest != null)) {
-            IngestManager.getDefault().scheduleDataSourceTasks(contentToIngest, moduleTemplates, ingestConfigPanel.getProcessUnallocSpace());
+            IngestManager.getDefault().scheduleDataSourceTasks(contentToIngest, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace());
         }
     }
 
@@ -209,7 +210,7 @@ public class IngestConfigurator {
         return csvList.toString();
     }
     
-    // RJCTODO: May need additional mappings
+    // RJCTODO: May need additional mappings - EWF Verify to EWF Verifier
     private HashSet<String> getModulesNamesFromSetting(String key, String defaultSetting) {
         // Get the ingest modules setting from the user's config file. 
         // If there is no such setting yet, create the default setting.

Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java

@@ -26,7 +26,7 @@ import org.openide.util.Lookup;
 import org.sleuthkit.autopsy.coreutils.Logger;
 
 /**
- * Looks up loaded ingest module factories using NetBean global lookup.
+ * Looks up loaded ingest module factories using the NetBean global lookup.
  */
 final class IngestModuleLoader {
 
@@ -35,7 +35,13 @@ final class IngestModuleLoader {
     private final List<IngestModuleFactory> moduleFactories = new ArrayList<>();
 
     private IngestModuleLoader() {
-        lookUpIngestModuleFactories();
+        // RJCTODO: Possibly add code to listen to changes in the collection and restore listener code...
+        // RJCTODO: Need a name uniqueness test/solution?
+        Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class);
+        for (IngestModuleFactory factory : factories) {
+            logger.log(Level.INFO, "Found ingest module factory: name = {0}, version = {1}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()});
+            moduleFactories.add(factory);
+        }
     }
 
     synchronized static IngestModuleLoader getInstance() {
@@ -48,13 +54,4 @@ final class IngestModuleLoader {
     List<IngestModuleFactory> getIngestModuleFactories() {
         return new ArrayList<>(moduleFactories);
     }
-
-    private void lookUpIngestModuleFactories() {
-        // RJCTODO: Possibly add code to listen to changes in the collection and restore listener code...
-        Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class);
-        for (IngestModuleFactory factory : factories) {
-            logger.log(Level.INFO, "Found ingest module factory: name = {0}, version = {1}", new Object[]{factory.getModuleDisplayName(), factory.getModuleVersionNumber()});
-            moduleFactories.add(factory);
-        }
-    }
 }

Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java

@@ -19,6 +19,7 @@
 package org.sleuthkit.autopsy.ingest;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Level;
@@ -71,6 +72,13 @@ final class IngestPipelinesConfiguration {
     }
 
     private void readPipelinesConfigurationFile() {
+        try {
+            PlatformUtil.extractResourceToUserConfigDir(IngestModuleLoader.class, PIPELINES_CONFIG_FILE);
+        } catch (IOException ex) {
+            logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir", ex);
+            return;
+        }
+        
         String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE;
         Document doc = XMLUtil.loadDoc(IngestModuleLoader.class, configFilePath, PIPELINES_CONFIG_FILE_XSD);
         if (doc == null) {

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java

@@ -75,7 +75,7 @@ import org.sleuthkit.datamodel.TskData.FileKnown;
  * on currently configured lists for ingest and writes results to blackboard
  * Reports interesting events to Inbox and to viewers
  *
- * Registered as a module in layer.xml
+ * Registered as a module in layer.xml RJCTODO: Track this down, does not seem to be true
  */
 public final class KeywordSearchIngestModule extends IngestModuleAdapter implements FileIngestModule {
 

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java

@@ -36,6 +36,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.autopsy.ingest.IngestModule.ResultCode;
 import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
+import org.sleuthkit.autopsy.ingest.IngestModuleContext;
 
 /**
  * Recent activity image ingest module
@@ -139,8 +140,7 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da
     }
 
     @Override
-    public void startUp(org.sleuthkit.autopsy.ingest.IngestModuleContext context) Exception {
+    public void startUp(IngestModuleContext context) throws Exception {
-        super.startUp(context);
         services = IngestServices.getDefault();
 
         Extract registry = new ExtractRegistry();
@@ -162,12 +162,7 @@ public final class RAImageIngestModule extends IngestModuleAdapter implements Da
         browserExtracters.add(iexplore);
         
        for (Extract extracter : extracters) {
-            try {
             extracter.init();
-            } catch (Exception ex) {
-                logger.log(Level.SEVERE, "Exception during init() of " + extracter.getName(), ex);
-                throw new IngestModuleException(ex.getMessage());
-            }
         }        
     }
 

ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java

@@ -26,13 +26,9 @@ import java.util.logging.Level;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
-import org.sleuthkit.autopsy.coreutils.Version;
 import org.sleuthkit.autopsy.ingest.FileIngestModule;
-import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile;
-import org.sleuthkit.autopsy.ingest.IngestModuleAbstractFile.ProcessResult;
 import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
 import org.sleuthkit.autopsy.ingest.IngestModuleContext;
-import org.sleuthkit.autopsy.ingest.IngestModuleInit;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleContentEvent;
 import org.sleuthkit.autopsy.scalpel.jni.ScalpelOutputParser.CarvedFileMeta;
@@ -52,7 +48,6 @@ import org.sleuthkit.datamodel.Volume;
 /**
  * Scalpel carving ingest module
  */
-
 class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileIngestModule {
     
     private static final Logger logger = Logger.getLogger(ScalpelCarverIngestModule.class.getName());
@@ -188,16 +183,13 @@ class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileInges
         }
         
         // reschedule carved files
-        
+        context.submitFilesForIngest(new ArrayList<AbstractFile>(carvedFiles));
-        for (LayoutFile carvedFile : carvedFiles) {
-            is.scheduleFile(carvedFile, pipelineContext);
-        }
         
         return ResultCode.OK;
     }
         
     @Override
-    public void startUp(IngestModuleContext context) throws IngestModuleException {        
+    public void startUp(IngestModuleContext context) throws Exception {        
         this.context = context;
         
         // make sure this is Windows

SevenZip/src/org/sleuthkit/autopsy/sevenzip/ArchiveFileExtractorModuleFactory.java

@@ -29,7 +29,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleSettings;
  * A factory for creating archive extractor file ingest modules and the user
  * interface panels used to configure the settings for instances of the modules.
  */
-@ServiceProvider(service = IngestModuleFactory.class)
+@ServiceProvider(service=IngestModuleFactory.class)
 public class ArchiveFileExtractorModuleFactory extends IngestModuleFactoryAdapter {
 
     static String getModuleName() {

SevenZip/src/org/sleuthkit/autopsy/sevenzip/SevenZipIngestModule.java

@@ -85,16 +85,17 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
     private static final int readHeaderSize = 4;
     private final byte[] fileHeaderBuffer = new byte[readHeaderSize];
     private static final int ZIP_SIGNATURE_BE = 0x504B0304;
+    private IngestModuleContext context;
 
     SevenZipIngestModule() {
     }
 
     @Override
-    public void startUp(IngestModuleContext context) throws IngestModuleException{
+    public void startUp(IngestModuleContext context) throws Exception{
-        super.startUp(context);
+        this.context = context;
-        unpackDir = getContext().getOutputDirectoryRelativePath();
+        unpackDir = context.getOutputDirectoryRelativePath();
-        unpackDirPath = getContext().getOutputDirectoryAbsolutePath();
+        unpackDirPath = context.getOutputDirectoryAbsolutePath();
-        fileManager = getContext().getCase().getServices().getFileManager();
+        fileManager = context.getCase().getServices().getFileManager();
 
         File unpackDirPathFile = new File(unpackDirPath);
         if (!unpackDirPathFile.exists()) {
@@ -104,7 +105,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
                 logger.log(Level.SEVERE, "Error initializing output dir: " + unpackDirPath, e);
                 String msg = "Error initializing archive extractor";
                 String details = "Error initializing output dir: " + unpackDirPath + ": " + e.getMessage();
-                getContext().postErrorIngestMessage(++messageID, msg, details);
+                context.postErrorIngestMessage(++messageID, msg, details);
                 throw e;
             }
         }
@@ -117,7 +118,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
             logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e);
             String msg = "Error initializing archive extractor";
             String details = "Could not initialize 7-ZIP library: " + e.getMessage();
-            getContext().postErrorIngestMessage(++messageID, msg, details);
+            context.postErrorIngestMessage(++messageID, msg, details);
             throw new RuntimeException(e);
         }
 
@@ -159,7 +160,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
         List<AbstractFile> unpackedFiles = unpack(abstractFile);
         if (!unpackedFiles.isEmpty()) {
             sendNewFilesEvent(abstractFile, unpackedFiles);
-            getContext().submitFilesForIngest(unpackedFiles);
+            context.submitFilesForIngest(unpackedFiles);
         }
 
         return ResultCode.OK;
@@ -226,7 +227,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
                         + ", item: " + itemName;
                 String details = "The archive item compression ratio is " + cRatio
                         + ", skipping processing of this archive item. ";
-                getContext().postWarningIngestMessage(++messageID, msg, details);
+                context.postWarningIngestMessage(++messageID, msg, details);
                 return true;
             } else {
                 return false;
@@ -257,7 +258,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
             String msg = "Possible ZIP bomb detected: " + archiveFile.getName();
             String details = "The archive is " + parentAr.getDepth()
                     + " levels deep, skipping processing of this archive and its contents ";
-            getContext().postWarningIngestMessage(++messageID, msg, details);
+            context.postWarningIngestMessage(++messageID, msg, details);
             return unpackedFiles;
         }
 
@@ -377,7 +378,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
                     if (newDiskSpace < MIN_FREE_DISK_SPACE) {
                         String msg = "Not enough disk space to unpack archive item: " + archiveFile.getName() + ", " + fileName;
                         String details = "The archive item is too large to unpack, skipping unpacking this item. ";
-                        getContext().postErrorIngestMessage(++messageID, msg, details);
+                        context.postErrorIngestMessage(++messageID, msg, details);
                         logger.log(Level.INFO, "Skipping archive item due not sufficient disk space for this item: {0}, {1}", new Object[]{archiveFile.getName(), fileName});
                         continue; //skip this file
                     } else {
@@ -472,7 +473,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
             String details = "Error unpacking ("
                     + (archiveFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC) ? "allocated" : "deleted") + ") " + fullName
                     + ". " + ex.getMessage();
-            getContext().postErrorIngestMessage(++messageID, msg, details);
+            context.postErrorIngestMessage(++messageID, msg, details);
         } finally {
             if (inArchive != null) {
                 try {
@@ -499,15 +500,15 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
             String encryptionType = fullEncryption ? ENCRYPTION_FULL : ENCRYPTION_FILE_LEVEL;
             try {
                 BlackboardArtifact artifact = archiveFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
-                artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), getContext().getModuleDisplayName(), encryptionType));
+                artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), ArchiveFileExtractorModuleFactory.getModuleName(), encryptionType));
-                getContext().fireDataEvent(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
+                context.fireDataEvent(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
             } catch (TskCoreException ex) {
                 logger.log(Level.SEVERE, "Error creating blackboard artifact for encryption detected for file: " + archiveFile, ex);
             }
 
             String msg = "Encrypted files in archive detected. ";
             String details = "Some files in archive: " + archiveFile.getName() + " are encrypted. Archive extractor was unable to extract all files from this archive.";
-            getContext().postWarningIngestMessage(++messageID, msg, details);
+            context.postWarningIngestMessage(++messageID, msg, details);
         }
 
         return unpackedFiles;
@@ -715,7 +716,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
             try {
                 DerivedFile df = fileManager.addDerivedFile(fileName, localRelPath, size,
                         node.getCtime(), node.getCrtime(), node.getAtime(), node.getMtime(),
-                        isFile, parent, "", getContext().getModuleDisplayName(), "", "");
+                        isFile, parent, "", ArchiveFileExtractorModuleFactory.getModuleName(), "", "");
                 node.setFile(df);
 
 

ewfVerify/src/org/sleuthkit/autopsy/ewfverify/EwfVerifierModuleFactory.java

@@ -37,7 +37,7 @@ public class EwfVerifierModuleFactory extends IngestModuleFactoryAdapter {
     }
 
     static String getModuleName() {
-        return "EWF Verify"; // RJCTODO: Is this what we want here?
+        return "EWF Verify"; // RJCTODO: Is this what we want here? Also, this class is not in pipeline config
     }
 
     @Override

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailParserModuleFactory.java

@@ -29,7 +29,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleSettings;
  * A factory for creating email parser file ingest modules and the user
  * interface panels used to configure the settings for instances of the modules.
  */
-@ServiceProvider(service = IngestModuleFactory.class)
+@ServiceProvider(service=IngestModuleFactory.class)
 public class EmailParserModuleFactory extends IngestModuleFactoryAdapter {
 
     static String getModuleName() {

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -53,8 +53,9 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
     private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
     private IngestServices services;
     private final String hashDBModuleName = "Hash Lookup";
-    private int messageId = 0;
+    private int messageId = 0; // RJCTODO: Not thread safe
     private FileManager fileManager;
+    private IngestModuleContext context;
 
     ThunderbirdMboxFileIngestModule() {
     }
@@ -72,16 +73,6 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
             return ResultCode.OK;
         }
 
-        // RJCTODO: We may be able to jettision this now
-        //file has read error, stop processing it
-        // @@@ I don't really like this
-        // we don't know if Hash was run or if it had lookup errors 
-//        IngestModuleAbstractFile.ResultCode hashDBResult =
-//                services.getAbstractFileModuleResult(hashDBModuleName);
-//        if (hashDBResult == IngestModuleAbstractFile.ResultCode.ERROR) {
-//            return ResultCode.ERROR;  
-//        }
-
         if (abstractFile.isVirtual()) {
             return ResultCode.OK;
         }
@@ -257,8 +248,8 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
     }
     
     @Override
-    public void startUp(IngestModuleContext context) {
+    public void startUp(IngestModuleContext context) throws Exception {
-        setContext(context);
+        this.context = context;
         services = IngestServices.getDefault();
         fileManager = Case.getCurrentCase().getServices().getFileManager();
     }
@@ -284,7 +275,7 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
                 services.fireModuleContentEvent(new ModuleContentEvent(derived));
             }
         }
-        getContext().submitFilesForIngest(derivedFiles);
+        context.submitFilesForIngest(derivedFiles);
         services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
     }