diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 59570a3347..3bd35249be 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -68,6 +68,10 @@ final class FileIngestPipeline { return modules.isEmpty(); } + /** + * Start up all of the modules in the pipeline. + * @return List of errors or empty list if no errors + */ List startUp() { List errors = new ArrayList<>(); for (FileIngestModuleDecorator module : modules) { @@ -80,6 +84,13 @@ final class FileIngestPipeline { return errors; } + /** + * Process the file down the pipeline of modules. + * Startup must have been called before this is called. + * + * @param file File to analyze + * @return List of errors or empty list if no errors + */ List process(AbstractFile file) { List errors = new ArrayList<>(); for (FileIngestModuleDecorator module : modules) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 6426323d26..d88abaed7f 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -21,6 +21,10 @@ package org.sleuthkit.autopsy.ingest; import java.util.Objects; import org.sleuthkit.datamodel.AbstractFile; +/** + * Represents a single file analysis task, which is defined + * by a file to analyze and the InjestJob/Pipeline to run it on. + */ final class FileIngestTask extends IngestTask { private final AbstractFile file; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index e3d7a9d081..5187da7f7b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -32,6 +32,10 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +/** + * InjestJobs store all settings and data associated with the user selecting a + * datasource and running a set of ingest modules on it. + */ final class IngestJob { private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); @@ -102,6 +106,11 @@ final class IngestJob { return processUnallocatedSpace; } + /** + * Create the file and data source pipelines. + * @param ingestModuleTemplates + * @throws InterruptedException + */ private void createIngestPipelines(List ingestModuleTemplates) throws InterruptedException { IngestJobContext context = new IngestJobContext(this); dataSourceIngestPipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates); @@ -118,6 +127,11 @@ final class IngestJob { return true; } + /** + * Start both the data source and file ingest pipelines + * @return + * @throws InterruptedException + */ private List start() throws InterruptedException { List errors = startUpIngestPipelines(); if (errors.isEmpty()) { @@ -142,6 +156,14 @@ final class IngestJob { return errors; } + + /** + * Startup each of the file and data source ingest modules to collect + * possible errors. + * + * @return + * @throws InterruptedException + */ private List startUpIngestPipelines() throws InterruptedException { List errors = new ArrayList<>(); errors.addAll(dataSourceIngestPipeline.startUp()); diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index e2ee1d7947..ce69602e39 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -44,7 +44,7 @@ import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; -import org.sleuthkit.datamodel.HashInfo; +import org.sleuthkit.datamodel.HashHitInfo; public class HashDbIngestModule implements FileIngestModule { private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName()); @@ -82,8 +82,8 @@ public class HashDbIngestModule implements FileIngestModule { @Override public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException { jobId = context.getJobId(); - getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); - getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets); + updateEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); + updateEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets); if (refCounter.incrementAndGet(jobId) == 1) { // if first module for this job then post error msgs if needed @@ -108,9 +108,14 @@ public class HashDbIngestModule implements FileIngestModule { } } - private void getEnabledHashSets(List hashSets, List enabledHashSets) { + /** + * Cycle through list of hashsets and return the subset that is enabled. + * @param allHashSets List of all hashsets from DB manager + * @param enabledHashSets List of enabled ones to return. + */ + private void updateEnabledHashSets(List allHashSets, List enabledHashSets) { enabledHashSets.clear(); - for (HashDb db : hashSets) { + for (HashDb db : allHashSets) { if (settings.isHashSetEnabled(db.getHashSetName())) { try { if (db.hasIndex()) { @@ -178,7 +183,7 @@ public class HashDbIngestModule implements FileIngestModule { for (HashDb db : knownBadHashSets) { try { long lookupstart = System.currentTimeMillis(); - HashInfo hashInfo = db.lookUp(file); + HashHitInfo hashInfo = db.lookupMD5(file); if (null != hashInfo) { foundBad = true; totals.totalKnownBadCount.incrementAndGet(); @@ -239,7 +244,7 @@ public class HashDbIngestModule implements FileIngestModule { for (HashDb db : knownHashSets) { try { long lookupstart = System.currentTimeMillis(); - if (db.hasMd5HashOf(file)) { + if (db.lookupMD5Quick(file)) { try { skCase.setKnown(file, TskData.FileKnown.KNOWN); break; diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java index 7aa31414be..894b330fee 100755 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java @@ -49,7 +49,7 @@ import org.netbeans.api.progress.ProgressHandleFactory; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.HashInfo; +import org.sleuthkit.datamodel.HashHitInfo; import org.sleuthkit.datamodel.HashEntry; import org.sleuthkit.datamodel.SleuthkitJNI; import org.sleuthkit.datamodel.TskCoreException; @@ -883,7 +883,13 @@ public class HashDbManager implements PropertyChangeListener { SleuthkitJNI.addToHashDatabase(hashes, handle); } - public boolean hasMd5HashOf(Content content) throws TskCoreException { + /** + * Perform a basic boolean lookup of the file's hash. + * @param content + * @return True if file's MD5 is in the hash database + * @throws TskCoreException + */ + public boolean lookupMD5Quick(Content content) throws TskCoreException { boolean result = false; assert content instanceof AbstractFile; if (content instanceof AbstractFile) { @@ -895,8 +901,14 @@ public class HashDbManager implements PropertyChangeListener { return result; } - public HashInfo lookUp(Content content) throws TskCoreException { - HashInfo result = null; + /** + * Lookup hash value in DB and provide details on file. + * @param content + * @return null if file is not in database. + * @throws TskCoreException + */ + public HashHitInfo lookupMD5(Content content) throws TskCoreException { + HashHitInfo result = null; // This only works for AbstractFiles and MD5 hashes at present. assert content instanceof AbstractFile; if (content instanceof AbstractFile) { @@ -907,6 +919,7 @@ public class HashDbManager implements PropertyChangeListener { } return result; } + boolean hasIndex() throws TskCoreException { return SleuthkitJNI.hashDatabaseHasLookupIndex(handle);