From f875da5c1257a6edb642e433dddf266cee013321 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 15 Jan 2025 16:41:50 -0500 Subject: [PATCH] updates for auto ingest --- .../autopsy/casemodule/ImageDSProcessor.java | 134 ++++++++++++------ .../casemodule/LocalDiskDSProcessor.java | 18 ++- .../commandlineingest/CommandLineCommand.java | 4 +- .../CommandLineIngestManager.java | 7 +- .../CommandLineOptionProcessor.java | 39 +++-- .../DataSourceProcessor.java | 46 ++++++ .../AutoIngestDataSource.java | 16 ++- .../AutoIngestDataSourceProcessor.java | 70 +++++++++ .../DataSourceProcessorUtility.java | 63 +++++++- .../autoingest/AutoIngestManager.java | 9 +- .../autoingest/AutopsyManifestFileParser.java | 6 +- .../experimental/autoingest/Manifest.java | 13 ++ 12 files changed, 348 insertions(+), 77 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java index 83bfb434f1..8f63d8459d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.logging.Level; import java.util.UUID; import javax.swing.filechooser.FileFilter; +import org.apache.commons.lang3.StringUtils; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; @@ -184,7 +185,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - run(null, progressMonitor, callback); + run(null, null, progressMonitor, callback); } /** @@ -204,9 +205,16 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour */ @Override public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, host, progressMonitor, callback); + } + + + @Override + public void run(String password, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { ingestStream = new DefaultIngestStream(); readConfigSettings(); this.host = host; + this.password = StringUtils.defaultString(password, this.password); try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId, this.password, this.host); @@ -220,6 +228,46 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour doAddImageProcess(deviceId, imagePath, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, this.password, progressMonitor, callback); } + + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the given settings instead of those provided by the + * selection and configuration panel. Returns as soon as the background task + * is started and uses the callback object to signal task completion and + * return results. + * + * @param deviceId An ASCII-printable identifier for the device + * associated with the data source that is + * intended to be unique across multiple cases + * (e.g., a UUID). + * @param imagePath Path to the image file. + * @param timeZone The time zone to use when processing dates + * and times for the image, obtained from + * java.util.TimeZone.getID. + * @param ignoreFatOrphanFiles Whether to parse orphans if the image has a + * FAT filesystem. + * @param progressMonitor Progress monitor for reporting progress + * during processing. + * @param callback Callback to call when processing is done. + */ + public void run(String deviceId, String imagePath, String timeZone, boolean ignoreFatOrphanFiles, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + ingestStream = new DefaultIngestStream(); + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{imagePath}, sectorSize, timeZone, "", "", "", deviceId, null, null); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + + doAddImageProcess(deviceId, imagePath, 0, timeZone, ignoreFatOrphanFiles, null, null, null, this.password, progressMonitor, callback); + } + + /** * Adds a data source to the case database using a background task in a @@ -241,7 +289,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour @Override public void runWithIngestStream(IngestJobSettings settings, DataSourceProcessorProgressMonitor progress, DataSourceProcessorCallback callBack) { - runWithIngestStream(null, settings, progress, callBack); + runWithIngestStream(null, null, settings, progress, callBack); } /** @@ -265,10 +313,18 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour @Override public void runWithIngestStream(Host host, IngestJobSettings settings, DataSourceProcessorProgressMonitor progress, DataSourceProcessorCallback callBack) { + runWithIngestStream(null, host, settings, progress, callBack); + } + + + @Override + public void runWithIngestStream(String password, Host host, IngestJobSettings settings, + DataSourceProcessorProgressMonitor progress, DataSourceProcessorCallback callBack) { // Read the settings from the wizard readConfigSettings(); this.host = host; + this.password = StringUtils.defaultIfEmpty(password, this.password); // Set up the data source before creating the ingest stream try { @@ -296,6 +352,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour doAddImageProcess(deviceId, imagePath, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, this.password, progress, callBack); } + /** * Store the options from the config panel. */ @@ -334,43 +391,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour return true; } - /** - * Adds a data source to the case database using a background task in a - * separate thread and the given settings instead of those provided by the - * selection and configuration panel. Returns as soon as the background task - * is started and uses the callback object to signal task completion and - * return results. - * - * @param deviceId An ASCII-printable identifier for the device - * associated with the data source that is - * intended to be unique across multiple cases - * (e.g., a UUID). - * @param imagePath Path to the image file. - * @param timeZone The time zone to use when processing dates - * and times for the image, obtained from - * java.util.TimeZone.getID. - * @param ignoreFatOrphanFiles Whether to parse orphans if the image has a - * FAT filesystem. - * @param progressMonitor Progress monitor for reporting progress - * during processing. - * @param callback Callback to call when processing is done. - */ - public void run(String deviceId, String imagePath, String timeZone, boolean ignoreFatOrphanFiles, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - ingestStream = new DefaultIngestStream(); - try { - image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), - new String[]{imagePath}, sectorSize, timeZone, "", "", "", deviceId, this.password, null); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); - final List errors = new ArrayList<>(); - errors.add(ex.getMessage()); - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); - return; - } - - doAddImageProcess(deviceId, imagePath, 0, timeZone, ignoreFatOrphanFiles, null, null, null, this.password, progressMonitor, callback); - } - + /** * Adds a data source to the case database using a background task in a * separate thread and the given settings instead of those provided by the @@ -472,6 +493,13 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour @Override public int canProcess(Path dataSourcePath) throws AutoIngestDataSourceProcessorException { + return canProcess(dataSourcePath, null); + } + + + + @Override + public int canProcess(Path dataSourcePath, String password) throws AutoIngestDataSourceProcessorException { // check file extension for supported types if (!isAcceptedByFiler(dataSourcePath.toFile(), filtersList)) { @@ -504,23 +532,29 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour @Override public void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { - process(deviceId, dataSourcePath, null, progressMonitor, callBack); + process(deviceId, dataSourcePath, this.password, null, progressMonitor, callBack); } @Override public void process(String deviceId, Path dataSourcePath, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + process(deviceId, dataSourcePath, this.password, host, progressMonitor, callBack); + } + + @Override + public void process(String deviceId, Path dataSourcePath, String password, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { // this method does not use the config panel this.deviceId = deviceId; this.imagePath = dataSourcePath.toString(); this.sectorSize = 0; this.timeZone = Calendar.getInstance().getTimeZone().getID(); + this.password = password; this.host = host; this.ignoreFatOrphanFiles = false; ingestStream = new DefaultIngestStream(); try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), - new String[]{imagePath}, sectorSize, timeZone, "", "", "", deviceId, this.password, host); + new String[]{imagePath}, sectorSize, timeZone, "", "", "", deviceId, password, host); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); final List errors = new ArrayList<>(); @@ -529,28 +563,36 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour return; } - doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, null, progressMonitor, callBack); + doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, password, progressMonitor, callBack); } + + @Override public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { - return processWithIngestStream(deviceId, dataSourcePath, null, settings, progressMonitor, callBack); + return processWithIngestStream(deviceId, dataSourcePath, this.password, null, settings, progressMonitor, callBack); } @Override public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, Host host, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + return processWithIngestStream(deviceId, dataSourcePath, this.password, host, settings, progressMonitor, callBack); + } + + @Override + public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, String password, Host host, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { // this method does not use the config panel this.deviceId = deviceId; this.imagePath = dataSourcePath.toString(); this.sectorSize = 0; this.timeZone = Calendar.getInstance().getTimeZone().getID(); this.host = host; + this.password = password; this.ignoreFatOrphanFiles = false; // Set up the data source before creating the ingest stream try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), - new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId, this.password, host); + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId, password, host); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); final List errors = new ArrayList<>(); @@ -570,8 +612,10 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour return null; } - doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, null, progressMonitor, callBack); + doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, password, progressMonitor, callBack); return ingestStream; } + + } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java index e63943a456..463d5c9bcd 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java @@ -136,9 +136,14 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - run(null, progressMonitor, callback); + run(null, null, progressMonitor, callback); } + @Override + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, host, progressMonitor, callback); + } + /** * Adds a data source to the case database using a background task in a * separate thread and the settings provided by the selection and @@ -155,7 +160,7 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { * to return results. */ @Override - public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + public void run(String password, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { deviceId = UUID.randomUUID().toString(); drivePath = configPanel.getContentPath(); sectorSize = configPanel.getSectorSize(); @@ -168,12 +173,13 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { } this.host = host; + this.password = password; Image image; try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), new String[]{drivePath}, sectorSize, - timeZone, null, null, null, deviceId, this.host); + timeZone, null, null, null, deviceId, this.password, this.host); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding local disk with path " + drivePath + " to database", ex); final List errors = new ArrayList<>(); @@ -183,7 +189,7 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { } addDiskTask = new AddImageTask( - new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings, password), + new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings, this.password), progressMonitor, new StreamingAddDataSourceCallbacks(new DefaultIngestStream()), new StreamingAddImageTaskCallback(new DefaultIngestStream(), callback)); @@ -242,7 +248,7 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), new String[]{drivePath}, sectorSize, - timeZone, null, null, null, deviceId); + timeZone, null, null, null, deviceId, this.password, null); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding local disk with path " + drivePath + " to database", ex); final List errors = new ArrayList<>(); @@ -251,7 +257,7 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { return; } - addDiskTask = new AddImageTask(new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings, password), + addDiskTask = new AddImageTask(new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings, this.password), progressMonitor, new StreamingAddDataSourceCallbacks(new DefaultIngestStream()), new StreamingAddImageTaskCallback(new DefaultIngestStream(), callback)); diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java index 1cacabe67c..378af77b75 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineCommand.java @@ -49,7 +49,9 @@ class CommandLineCommand { DATA_SOURCE_PATH, DATA_SOURCE_ID, INGEST_PROFILE_NAME, - REPORT_PROFILE_NAME; + REPORT_PROFILE_NAME, + BITLOCKER_KEY, + ; } private final CommandType type; diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java index ffc31fff1c..64940a022d 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java @@ -198,7 +198,8 @@ public class CommandLineIngestManager extends CommandLineManager { } String dataSourcePath = inputs.get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); - dataSource = new AutoIngestDataSource(UUID.randomUUID().toString(), Paths.get(dataSourcePath)); + String password = inputs.get(CommandLineCommand.InputType.BITLOCKER_KEY.name()); + dataSource = new AutoIngestDataSource(UUID.randomUUID().toString(), Paths.get(dataSourcePath), password); runDataSourceProcessor(caseForJob, dataSource); String outputDirPath = getOutputDirPath(caseForJob); @@ -406,7 +407,7 @@ public class CommandLineIngestManager extends CommandLineManager { // Get an ordered list of data source processors to try List validDataSourceProcessors; try { - validDataSourceProcessors = DataSourceProcessorUtility.getOrderedListOfDataSourceProcessors(dataSource.getPath()); + validDataSourceProcessors = DataSourceProcessorUtility.getOrderedListOfDataSourceProcessors(dataSource.getPath(), dataSource.getPassword()); } catch (AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException ex) { LOGGER.log(Level.SEVERE, "Exception while determining best data source processor for {0}", dataSource.getPath()); // rethrow the exception. @@ -429,7 +430,7 @@ public class CommandLineIngestManager extends CommandLineManager { DataSourceProcessorCallback callBack = new AddDataSourceCallback(caseForJob, dataSource, taskId, ingestLock); caseForJob.notifyAddingDataSource(taskId); LOGGER.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{dataSource.getPath(), selectedProcessor.getDataSourceType()}); - selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), dataSource.getPassword(), null, progressMonitor, callBack); ingestLock.wait(); // at this point we got the content object(s) from the current DSP. diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java index 921c2e0a65..1d4b55f49a 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.lang3.ArrayUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.netbeans.api.sendopts.CommandException; import org.netbeans.spi.sendopts.Env; @@ -52,6 +53,7 @@ public class CommandLineOptionProcessor extends OptionProcessor { private final Option dataSourcePathOption = Option.requiredArgument('s', "dataSourcePath"); private final Option dataSourceObjectIdOption = Option.requiredArgument('i', "dataSourceObjectId"); private final Option addDataSourceCommandOption = Option.withoutArgument('a', "addDataSource"); + private final Option bitlockerKeyCommandOption = Option.withoutArgument('k', "key"); private final Option runIngestCommandOption = Option.optionalArgument('r', "runIngest"); private final Option listAllDataSourcesCommandOption = Option.withoutArgument('l', "listAllDataSources"); private final Option generateReportsOption = Option.optionalArgument('g', "generateReports"); @@ -81,20 +83,21 @@ public class CommandLineOptionProcessor extends OptionProcessor { @Override protected Set