diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java index a1443bde31..53320d9313 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java @@ -30,6 +30,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgress import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.imagewriter.ImageWriterService; import org.sleuthkit.autopsy.imagewriter.ImageWriterSettings; +import org.sleuthkit.datamodel.AddDataSourceCallbacks; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitJNI; @@ -42,17 +43,10 @@ import org.sleuthkit.datamodel.TskDataException; class AddImageTask implements Runnable { private final Logger logger = Logger.getLogger(AddImageTask.class.getName()); - private final String deviceId; - private final String imagePath; - private final int sectorSize; - private final String timeZone; - private final ImageWriterSettings imageWriterSettings; - private final boolean ignoreFatOrphanFiles; - private final String md5; - private final String sha1; - private final String sha256; + private final ImageDetails imageDetails; private final DataSourceProcessorProgressMonitor progressMonitor; - private final DataSourceProcessorCallback callback; + private final AddDataSourceCallbacks addDataSourceCallbacks; + private final AddImageTaskCallback addImageTaskCallback; private boolean criticalErrorOccurred; /* @@ -73,40 +67,18 @@ class AddImageTask implements Runnable { /** * Constructs a runnable task that adds an image to the case database. - * - * @param deviceId An ASCII-printable identifier for the device - * associated with the data source that is - * intended to be unique across multiple cases - * (e.g., a UUID). - * @param imagePath Path to the image file. - * @param sectorSize The sector size (use '0' for autodetect). - * @param timeZone The time zone to use when processing dates - * and times for the image, obtained from - * java.util.TimeZone.getID. - * @param ignoreFatOrphanFiles Whether to parse orphans if the image has a - * FAT filesystem. - * @param md5 The MD5 hash of the image, may be null. - * @param sha1 The SHA-1 hash of the image, may be null. - * @param sha256 The SHA-256 hash of the image, may be null. - * @param imageWriterPath Path that a copy of the image should be - * written to. Use empty string to disable image - * writing + * + * @param imageDetails Holds all data about the image. * @param progressMonitor Progress monitor to report progress during * processing. - * @param callback Callback to call when processing is done. + * @param addDataSourceCallbacks Callback for sending data to the ingest pipeline if an ingest stream is being used. + * @param addImageTaskCallback Callback for dealing with add image task completion. */ - AddImageTask(String deviceId, String imagePath, int sectorSize, String timeZone, boolean ignoreFatOrphanFiles, String md5, String sha1, String sha256, ImageWriterSettings imageWriterSettings, - DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - this.deviceId = deviceId; - this.imagePath = imagePath; - this.sectorSize = sectorSize; - this.timeZone = timeZone; - this.ignoreFatOrphanFiles = ignoreFatOrphanFiles; - this.md5 = md5; - this.sha1 = sha1; - this.sha256 = sha256; - this.imageWriterSettings = imageWriterSettings; - this.callback = callback; + AddImageTask(ImageDetails imageDetails, DataSourceProcessorProgressMonitor progressMonitor, AddDataSourceCallbacks addDataSourceCallbacks, + AddImageTaskCallback addImageTaskCallback) { + this.imageDetails = imageDetails; + this.addDataSourceCallbacks = addDataSourceCallbacks; + this.addImageTaskCallback = addImageTaskCallback; this.progressMonitor = progressMonitor; tskAddImageProcessLock = new Object(); } @@ -120,21 +92,21 @@ class AddImageTask implements Runnable { try { currentCase = Case.getCurrentCaseThrows(); } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, String.format("Failed to add image data source at %s, no current case", imagePath), ex); + logger.log(Level.SEVERE, String.format("Failed to start AddImageTask for %s, no current case", imageDetails.getImagePath()), ex); return; } progressMonitor.setIndeterminate(true); progressMonitor.setProgress(0); String imageWriterPath = ""; - if (imageWriterSettings != null) { - imageWriterPath = imageWriterSettings.getPath(); + if (imageDetails.imageWriterSettings != null) { + imageWriterPath = imageDetails.imageWriterSettings.getPath(); } List errorMessages = new ArrayList<>(); List newDataSources = new ArrayList<>(); try { synchronized (tskAddImageProcessLock) { if (!tskAddImageProcessStopped) { - tskAddImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(timeZone, true, ignoreFatOrphanFiles, imageWriterPath); + tskAddImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(imageDetails.timeZone, true, imageDetails.ignoreFatOrphanFiles, imageWriterPath); } else { return; } @@ -143,7 +115,7 @@ class AddImageTask implements Runnable { progressUpdateThread.start(); runAddImageProcess(errorMessages); progressUpdateThread.interrupt(); - commitOrRevertAddImageProcess(currentCase, errorMessages, newDataSources); + finishAddImageProcess(errorMessages, newDataSources); progressMonitor.setProgress(100); } finally { DataSourceProcessorCallback.DataSourceProcessorResult result; @@ -154,7 +126,7 @@ class AddImageTask implements Runnable { } else { result = DataSourceProcessorResult.NO_ERRORS; } - callback.done(result, errorMessages, newDataSources); + addImageTaskCallback.onCompleted(result, errorMessages, newDataSources); } } @@ -177,7 +149,7 @@ class AddImageTask implements Runnable { tskAddImageProcess.stop(); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Error cancelling adding image %s to the case database", imagePath), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error cancelling adding image %s to the case database", imageDetails.getImagePath()), ex); //NON-NLS } } } @@ -191,23 +163,22 @@ class AddImageTask implements Runnable { */ private void runAddImageProcess(List errorMessages) { try { - tskAddImageProcess.run(deviceId, new String[]{imagePath}, sectorSize); + tskAddImageProcess.run(imageDetails.deviceId, imageDetails.image, imageDetails.sectorSize, this.addDataSourceCallbacks); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Critical error occurred adding image %s", imagePath), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Critical error occurred adding image %s", imageDetails.getImagePath()), ex); //NON-NLS criticalErrorOccurred = true; errorMessages.add(ex.getMessage()); } catch (TskDataException ex) { - logger.log(Level.WARNING, String.format("Non-critical error occurred adding image %s", imagePath), ex); //NON-NLS + logger.log(Level.WARNING, String.format("Non-critical error occurred adding image %s", imageDetails.getImagePath()), ex); //NON-NLS errorMessages.add(ex.getMessage()); } } /** - * Commits or reverts the results of the TSK add image process. If the - * process was stopped before it completed or there was a critical error the - * results are reverted, otherwise they are committed. + * Handle the results of the TSK add image process. + * The image will be in the database even if a critical error occurred or + * the user canceled. * - * @param currentCase The current case. * @param errorMessages Error messages, if any, are added to this list for * eventual return via the callback. * @param newDataSources If the new image is successfully committed, it is @@ -216,84 +187,66 @@ class AddImageTask implements Runnable { * * @return */ - private void commitOrRevertAddImageProcess(Case currentCase, List errorMessages, List newDataSources) { + private void finishAddImageProcess(List errorMessages, List newDataSources) { synchronized (tskAddImageProcessLock) { - if (tskAddImageProcessStopped || criticalErrorOccurred) { + Image newImage = imageDetails.image; + String verificationError = newImage.verifyImageSize(); + if (!verificationError.isEmpty()) { + errorMessages.add(verificationError); + } + if (imageDetails.imageWriterSettings != null) { + ImageWriterService.createImageWriter(newImage.getId(), imageDetails.imageWriterSettings); + } + newDataSources.add(newImage); + + // If the add image process was cancelled don't do any further processing here + if (tskAddImageProcessStopped) { + return; + } + + if (!StringUtils.isBlank(imageDetails.md5)) { try { - tskAddImageProcess.revert(); + newImage.setMD5(imageDetails.md5); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Error reverting after adding image %s to the case database", imagePath), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Failed to add MD5 hash for image data source %s (objId=%d)", newImage.getName(), newImage.getId()), ex); errorMessages.add(ex.getMessage()); criticalErrorOccurred = true; + } catch (TskDataException ignored) { + /* + * The only reasonable way for this to happen at + * present is through C/C++ processing of an EWF + * image, which is not an error. + */ } - } else { + } + if (!StringUtils.isBlank(imageDetails.sha1)) { try { - long imageId = tskAddImageProcess.commit(); - if (imageId != 0) { - Image newImage = currentCase.getSleuthkitCase().getImageById(imageId); - String verificationError = newImage.verifyImageSize(); - if (!verificationError.isEmpty()) { - errorMessages.add(verificationError); - } - if (imageWriterSettings != null) { - ImageWriterService.createImageWriter(imageId, imageWriterSettings); - } - newDataSources.add(newImage); - if (!StringUtils.isBlank(md5)) { - try { - newImage.setMD5(md5); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Failed to add MD5 hash for image data source %s (objId=%d)", newImage.getName(), newImage.getId()), ex); - errorMessages.add(ex.getMessage()); - criticalErrorOccurred = true; - } catch (TskDataException ignored) { - /* - * The only reasonable way for this to happen at - * present is through C/C++ processing of an EWF - * image, which is not an error. - */ - } - } - if (!StringUtils.isBlank(sha1)) { - try { - newImage.setSha1(sha1); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Failed to add SHA1 hash for image data source %s (objId=%d)", newImage.getName(), newImage.getId()), ex); - errorMessages.add(ex.getMessage()); - criticalErrorOccurred = true; - } catch (TskDataException ignored) { - /* - * The only reasonable way for this to happen at - * present is through C/C++ processing of an EWF - * image, which is not an error. - */ - } - } - if (!StringUtils.isBlank(sha256)) { - try { - newImage.setSha256(sha256); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Failed to add SHA256 for image data source %s (objId=%d)", newImage.getName(), newImage.getId()), ex); - errorMessages.add(ex.getMessage()); - criticalErrorOccurred = true; - } catch (TskDataException ignored) { - /* - * The only reasonable way for this to happen at - * present is through C/C++ processing of an EWF - * image, which is not an error. - */ - } - } - } else { - String errorMessage = String.format("Error commiting after adding image %s to the case database, no object id returned", imagePath); //NON-NLS - logger.log(Level.SEVERE, errorMessage); - errorMessages.add(errorMessage); - criticalErrorOccurred = true; - } + newImage.setSha1(imageDetails.sha1); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Error committing adding image %s to the case database", imagePath), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Failed to add SHA1 hash for image data source %s (objId=%d)", newImage.getName(), newImage.getId()), ex); errorMessages.add(ex.getMessage()); criticalErrorOccurred = true; + } catch (TskDataException ignored) { + /* + * The only reasonable way for this to happen at + * present is through C/C++ processing of an EWF + * image, which is not an error. + */ + } + } + if (!StringUtils.isBlank(imageDetails.sha256)) { + try { + newImage.setSha256(imageDetails.sha256); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Failed to add SHA256 for image data source %s (objId=%d)", newImage.getName(), newImage.getId()), ex); + errorMessages.add(ex.getMessage()); + criticalErrorOccurred = true; + } catch (TskDataException ignored) { + /* + * The only reasonable way for this to happen at + * present is through C/C++ processing of an EWF + * image, which is not an error. + */ } } } @@ -352,4 +305,37 @@ class AddImageTask implements Runnable { } } + /** + * Utility class to hold image data. + */ + static class ImageDetails { + String deviceId; + Image image; + int sectorSize; + String timeZone; + boolean ignoreFatOrphanFiles; + String md5; + String sha1; + String sha256; + ImageWriterSettings imageWriterSettings; + + ImageDetails(String deviceId, Image image, int sectorSize, String timeZone, boolean ignoreFatOrphanFiles, String md5, String sha1, String sha256, ImageWriterSettings imageWriterSettings) { + this.deviceId = deviceId; + this.image = image; + this.sectorSize = sectorSize; + this.timeZone = timeZone; + this.ignoreFatOrphanFiles = ignoreFatOrphanFiles; + this.md5 = md5; + this.sha1 = sha1; + this.sha256 = sha256; + this.imageWriterSettings = imageWriterSettings; + } + + String getImagePath() { + if (image.getPaths().length > 0) { + return image.getPaths()[0]; + } + return "Unknown data source path"; + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardAddingProgressPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardAddingProgressPanel.java index 15eb0c3c67..2b880093e9 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardAddingProgressPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardAddingProgressPanel.java @@ -302,7 +302,9 @@ class AddImageWizardAddingProgressPanel extends ShortcutWizardDescriptorPanel { private void startIngest() { if (!newContents.isEmpty() && readyToIngest && !ingested) { ingested = true; - IngestManager.getInstance().queueIngestJob(newContents, ingestJobSettings); + if (dsProcessor != null && ! dsProcessor.supportsIngestStream()) { + IngestManager.getInstance().queueIngestJob(newContents, ingestJobSettings); + } setStateFinished(); } } @@ -360,8 +362,12 @@ class AddImageWizardAddingProgressPanel extends ShortcutWizardDescriptorPanel { setStateStarted(); - // Kick off the DSProcessor - dsProcessor.run(getDSPProgressMonitorImpl(), cbObj); + // Kick off the DSProcessor + if (dsProcessor.supportsIngestStream()) { + dsProcessor.runWithIngestStream(ingestJobSettings, getDSPProgressMonitorImpl(), cbObj); + } else { + dsProcessor.run(getDSPProgressMonitorImpl(), cbObj); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIterator.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIterator.java index 9dad21446e..a79e5074fe 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIterator.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIterator.java @@ -41,6 +41,7 @@ class AddImageWizardIterator implements WizardDescriptor.Iterator allExt = new ArrayList<>(); private static final GeneralFilter rawFilter = new GeneralFilter(GeneralFilter.RAW_IMAGE_EXTS, GeneralFilter.RAW_IMAGE_DESC); private static final GeneralFilter encaseFilter = new GeneralFilter(GeneralFilter.ENCASE_IMAGE_EXTS, GeneralFilter.ENCASE_IMAGE_DESC); @@ -58,6 +67,8 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour private static final List filtersList = new ArrayList<>(); private final ImageFilePanel configPanel; private AddImageTask addImageTask; + private IngestStream ingestStream = null; + private Image image = null; /* * TODO: Remove the setDataSourceOptionsCalled flag and the settings fields * when the deprecated method setDataSourceOptions is removed. @@ -170,6 +181,68 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + ingestStream = new DefaultIngestStream(); + readConfigSettings(); + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + + doAddImageProcess(deviceId, imagePath, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Files found during ingest will be sent directly to the + * IngestStream provided. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true, and + * should only be called for DSPs that support ingest streams. + * + * @param settings The ingest job settings. + * @param progress Progress monitor that will be used by the + * background task to report progress. + * @param callBack Callback that will be used by the background task + * to return results. + */ + @Override + public void runWithIngestStream(IngestJobSettings settings, DataSourceProcessorProgressMonitor progress, + DataSourceProcessorCallback callBack) { + + // Read the settings from the wizard + readConfigSettings(); + + // Set up the data source before creating the ingest stream + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + + // Now initialize the ingest stream + this.ingestStream = IngestManager.getInstance().openIngestStream(image, settings); + + doAddImageProcess(deviceId, imagePath, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, progress, callBack); + } + + /** + * Store the options from the config panel. + */ + private void readConfigSettings() { if (!setDataSourceOptionsCalled) { configPanel.storeSettings(); deviceId = UUID.randomUUID().toString(); @@ -190,8 +263,17 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour sha256 = null; } } - run(deviceId, imagePath, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, progressMonitor, callback); } + + /** + * Check if this DSP supports ingest streams. + * + * @return True if this DSP supports an ingest stream, false otherwise. + */ + @Override + public boolean supportsIngestStream() { + return true; + } /** * Adds a data source to the case database using a background task in a @@ -215,7 +297,19 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour * @param callback Callback to call when processing is done. */ public void run(String deviceId, String imagePath, String timeZone, boolean ignoreFatOrphanFiles, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - run(deviceId, imagePath, 0, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callback); + ingestStream = new DefaultIngestStream(); + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{imagePath}, sectorSize, timeZone, "", "", "", deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + + doAddImageProcess(deviceId, imagePath, 0, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callback); } /** @@ -224,6 +318,10 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour * selection and configuration panel. Returns as soon as the background task * is started and uses the callback object to signal task completion and * return results. + * + * The image should be loaded in the database and stored in "image" + * before calling this method. Additionally, an ingest stream should be initialized + * and stored in "ingestStream". * * @param deviceId An ASCII-printable identifier for the device * associated with the data source that is @@ -243,8 +341,31 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour * during processing. * @param callback Callback to call when processing is done. */ - private void run(String deviceId, String imagePath, int sectorSize, String timeZone, boolean ignoreFatOrphanFiles, String md5, String sha1, String sha256, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - addImageTask = new AddImageTask(deviceId, imagePath, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, null, progressMonitor, callback); + private void doAddImageProcess(String deviceId, String imagePath, int sectorSize, String timeZone, boolean ignoreFatOrphanFiles, String md5, String sha1, String sha256, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + + // If the data source or ingest stream haven't been initialized, stop processing + if (ingestStream == null) { + String message = "Ingest stream was not initialized before running the add image process on " + imagePath; + logger.log(Level.SEVERE, message); + final List errors = new ArrayList<>(); + errors.add(message); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + if (image == null) { + String message = "Image was not added to database before running the add image process on " + imagePath; + logger.log(Level.SEVERE, message); + final List errors = new ArrayList<>(); + errors.add(message); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + + AddImageTask.ImageDetails imageDetails = new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, null); + addImageTask = new AddImageTask(imageDetails, + progressMonitor, + new StreamingAddDataSourceCallbacks(ingestStream), + new StreamingAddImageTaskCallback(ingestStream, callback)); new Thread(addImageTask).start(); } @@ -260,6 +381,9 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour if (null != addImageTask) { addImageTask.cancelTask(); } + if (ingestStream != null) { + ingestStream.stop(); + } } /** @@ -316,7 +440,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour this.timeZone = Calendar.getInstance().getTimeZone().getID(); this.ignoreFatOrphanFiles = false; setDataSourceOptionsCalled = true; - run(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); + doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); } /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java index 60c0aed94b..0551d15466 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java @@ -18,15 +18,22 @@ */ package org.sleuthkit.autopsy.casemodule; +import java.util.ArrayList; import java.util.Calendar; +import java.util.List; import java.util.UUID; +import java.util.logging.Level; import javax.swing.JPanel; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.imagewriter.ImageWriterSettings; +import org.sleuthkit.datamodel.Image; +import org.sleuthkit.datamodel.SleuthkitJNI; +import org.sleuthkit.datamodel.TskCoreException; /** * A local drive data source processor that implements the DataSourceProcessor @@ -37,6 +44,7 @@ import org.sleuthkit.autopsy.imagewriter.ImageWriterSettings; @ServiceProvider(service = DataSourceProcessor.class) public class LocalDiskDSProcessor implements DataSourceProcessor { + private final Logger logger = Logger.getLogger(LocalDiskDSProcessor.class.getName()); private static final String DATA_SOURCE_TYPE = NbBundle.getMessage(LocalDiskDSProcessor.class, "LocalDiskDSProcessor.dsType.text"); private final LocalDiskPanel configPanel; private AddImageTask addDiskTask; @@ -139,7 +147,25 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { imageWriterSettings = null; } } - addDiskTask = new AddImageTask(deviceId, drivePath, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings, progressMonitor, callback); + + Image image; + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{drivePath}, sectorSize, + timeZone, null, null, null, deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding local disk with path " + drivePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + + addDiskTask = new AddImageTask( + new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings), + progressMonitor, + new StreamingAddDataSourceCallbacks(new DefaultIngestStream()), + new StreamingAddImageTaskCallback(new DefaultIngestStream(), callback)); new Thread(addDiskTask).start(); } @@ -191,7 +217,23 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { * @param callback Callback to call when processing is done. */ private void run(String deviceId, String drivePath, int sectorSize, String timeZone, boolean ignoreFatOrphanFiles, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - addDiskTask = new AddImageTask(deviceId, drivePath, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings, progressMonitor, callback); + Image image; + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{drivePath}, sectorSize, + timeZone, null, null, null, deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding local disk with path " + drivePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return; + } + + addDiskTask = new AddImageTask(new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings), + progressMonitor, + new StreamingAddDataSourceCallbacks(new DefaultIngestStream()), + new StreamingAddImageTaskCallback(new DefaultIngestStream(), callback)); new Thread(addDiskTask).start(); } diff --git a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java index 8e81590231..de51cf8add 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.corecomponentinterfaces; import javax.swing.JPanel; +import org.sleuthkit.autopsy.ingest.IngestJobSettings; /** * Interface implemented by classes that add data sources of a particular type @@ -36,10 +37,6 @@ import javax.swing.JPanel; * * Data source processors should perform all processing in a background task in * a separate thread, reporting results using a callback object. - * - * It is recommended that implementers provide an overload of the run method - * that allows the data source processor to be run independently of the - * selection and configuration panel. */ public interface DataSourceProcessor { @@ -111,6 +108,38 @@ public interface DataSourceProcessor { * to return results. */ void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback); + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Files found during ingest will be sent directly to + * the IngestStream provided. Returns as soon as the background task is + * started. The background task uses a callback object to signal task + * completion and return results. + * + * This method should not be called unless isPanelValid returns true, and + * should only be called for DSPs that support ingest streams. The ingest + * settings must be complete before calling this method. + * + * @param settings The ingest job settings. + * @param progress Progress monitor that will be used by the background task + * to report progress. + * @param callBack Callback that will be used by the background task to + * return results. + */ + default void runWithIngestStream(IngestJobSettings settings, DataSourceProcessorProgressMonitor progress, + DataSourceProcessorCallback callBack) { + throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor"); + } + + /** + * Check if this DSP supports ingest streams. + * + * @return True if this DSP supports an ingest stream, false otherwise. + */ + default boolean supportsIngestStream() { + return false; + } /** * Requests cancellation of the background task that adds a data source to diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java deleted file mode 100644 index 4527892323..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ /dev/null @@ -1,1439 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2014-2019 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import javax.swing.JOptionPane; -import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Cancellable; -import org.openide.util.NbBundle; -import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.NetworkUtils; -import org.sleuthkit.autopsy.ingest.DataSourceIngestPipeline.PipelineModule; -import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason; -import org.sleuthkit.autopsy.ingest.IngestTasksScheduler.IngestJobTasksSnapshot; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.IngestJobInfo; -import org.sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType; -import org.sleuthkit.datamodel.IngestModuleInfo; -import org.sleuthkit.datamodel.IngestModuleInfo.IngestModuleType; -import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; -import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer; - -/** - * Encapsulates a data source and the ingest module pipelines used to process - * it. - */ -public final class DataSourceIngestJob { - - private static String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; - - private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName()); - - // to match something like: "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" - private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); - - /** - * These fields define a data source ingest job: the parent ingest job, an - * ID, the user's ingest job settings, and the data source to be analyzed. - * Optionally, there is a set of files to be analyzed instead of analyzing - * all of the files in the data source. - */ - private final IngestJob parentJob; - private static final AtomicLong nextJobId = new AtomicLong(0L); - private final long id; - private final IngestJobSettings settings; - private final Content dataSource; - private final List files = new ArrayList<>(); - - /** - * A data source ingest job runs in stages. - */ - private static enum Stages { - - /** - * Setting up for processing. - */ - INITIALIZATION, - /** - * Running high priority data source level ingest modules and file level - * ingest modules. - */ - FIRST, - /** - * Running lower priority, usually long-running, data source level - * ingest modules. - */ - SECOND, - /** - * Cleaning up. - */ - FINALIZATION - }; - private volatile Stages stage = DataSourceIngestJob.Stages.INITIALIZATION; - private final Object stageCompletionCheckLock = new Object(); - - /** - * A data source ingest job has separate data source level ingest module - * pipelines for the first and second processing stages. Longer running, - * lower priority modules belong in the second stage pipeline, although this - * cannot be enforced. Note that the pipelines for both stages are created - * at job start up to allow for verification that they both can be started - * up without errors. - */ - private final Object dataSourceIngestPipelineLock = new Object(); - private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; - private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; - - /** - * A data source ingest job has a collection of identical file level ingest - * module pipelines, one for each file level ingest thread in the ingest - * manager. A blocking queue is used to dole out the pipelines to the - * threads and an ordinary list is used when the ingest job needs to access - * the pipelines to query their status. - */ - private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); - private final List fileIngestPipelines = new ArrayList<>(); - - /** - * A data source ingest job supports cancellation of either the currently - * running data source level ingest module or the entire ingest job. - * - * TODO: The currentDataSourceIngestModuleCancelled field and all of the - * code concerned with it is a hack to avoid an API change. The next time an - * API change is legal, a cancel() method needs to be added to the - * IngestModule interface and this field should be removed. The "ingest job - * is canceled" queries should also be removed from the IngestJobContext - * class. - */ - private volatile boolean currentDataSourceIngestModuleCancelled; - private final List cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>(); - private volatile boolean cancelled; - private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED; - - /** - * A data source ingest job uses the task scheduler singleton to create and - * queue the ingest tasks that make up the job. - */ - private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - - /** - * A data source ingest job can run interactively using NetBeans progress - * handles. - */ - private final boolean doUI; - - /** - * A data source ingest job uses these fields to report data source level - * ingest progress. - */ - private final Object dataSourceIngestProgressLock = new Object(); - private ProgressHandle dataSourceIngestProgress; - - /** - * A data source ingest job uses these fields to report file level ingest - * progress. - */ - private final Object fileIngestProgressLock = new Object(); - private final List filesInProgress = new ArrayList<>(); - private long estimatedFilesToProcess; - private long processedFiles; - private ProgressHandle fileIngestProgress; - private String currentFileIngestModule = ""; - private String currentFileIngestTask = ""; - private final List ingestModules = new ArrayList<>(); - private volatile IngestJobInfo ingestJob; - - /** - * A data source ingest job uses this field to report its creation time. - */ - private final long createTime; - - /** - * Constructs an object that encapsulates a data source and the ingest - * module pipelines used to analyze it. - * - * @param parentJob The ingest job of which this data source ingest - * job is a part. - * @param dataSource The data source to be ingested. - * @param settings The settings for the ingest job. - * @param runInteractively Whether or not this job should use NetBeans - * progress handles. - */ - DataSourceIngestJob(IngestJob parentJob, Content dataSource, IngestJobSettings settings, boolean runInteractively) { - this(parentJob, dataSource, Collections.emptyList(), settings, runInteractively); - } - - /** - * Constructs an object that encapsulates a data source and the ingest - * module pipelines used to analyze it. Either all of the files in the data - * source or a given subset of the files will be analyzed. - * - * @param parentJob The ingest job of which this data source ingest - * job is a part. - * @param dataSource The data source to be ingested. - * @param files A subset of the files for the data source. - * @param settings The settings for the ingest job. - * @param runInteractively Whether or not this job should use NetBeans - * progress handles. - */ - DataSourceIngestJob(IngestJob parentJob, Content dataSource, List files, IngestJobSettings settings, boolean runInteractively) { - this.parentJob = parentJob; - this.id = DataSourceIngestJob.nextJobId.getAndIncrement(); - this.dataSource = dataSource; - this.files.addAll(files); - this.settings = settings; - this.doUI = runInteractively; - this.createTime = new Date().getTime(); - this.createIngestPipelines(); - } - - /** - * Adds ingest modules to a list with autopsy modules first and third party - * modules next. - * - * @param dest The destination for the modules to be added. - * @param src A map of fully qualified class name mapped to the - * IngestModuleTemplate. - * @param jythonSrc A map of fully qualified class name mapped to the - * IngestModuleTemplate for jython modules. - */ - private static void addOrdered(final List dest, - final Map src, final Map jythonSrc) { - - final List autopsyModules = new ArrayList<>(); - final List thirdPartyModules = new ArrayList<>(); - - Stream.concat(src.entrySet().stream(), jythonSrc.entrySet().stream()).forEach((templateEntry) -> { - if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) { - autopsyModules.add(templateEntry.getValue()); - } else { - thirdPartyModules.add(templateEntry.getValue()); - } - }); - - dest.addAll(autopsyModules); - dest.addAll(thirdPartyModules); - } - - /** - * Takes a classname like - * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" - * and provides "GPX_Parser_Module.GPXParserFileIngestModuleFactory" or null - * if not in jython package. - * - * @param canonicalName The canonical name. - * - * @return The jython name or null if not in jython package. - */ - private static String getJythonName(String canonicalName) { - Matcher m = JYTHON_REGEX.matcher(canonicalName); - if (m.find()) { - return String.format("%s.%s", m.group(1), m.group(2)); - } else { - return null; - } - } - - /** - * Adds a template to the appropriate map. If the class is a jython class, - * then it is added to the jython map. Otherwise, it is added to the - * mapping. - * - * @param mapping Mapping for non-jython objects. - * @param jythonMapping Mapping for jython objects. - * @param template The template to add. - */ - private static void addModule(Map mapping, - Map jythonMapping, IngestModuleTemplate template) { - - String className = template.getModuleFactory().getClass().getCanonicalName(); - String jythonName = getJythonName(className); - if (jythonName != null) { - jythonMapping.put(jythonName, template); - } else { - mapping.put(className, template); - } - } - - /** - * Creates the file and data source ingest pipelines. - */ - private void createIngestPipelines() { - List ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates(); - - /** - * Make mappings of ingest module factory class names to templates. - */ - Map dataSourceModuleTemplates = new LinkedHashMap<>(); - Map fileModuleTemplates = new LinkedHashMap<>(); - - // mappings for jython modules. These mappings are only used to determine modules in the pipelineconfig.xml. - - Map jythonDataSourceModuleTemplates = new LinkedHashMap<>(); - Map jythonFileModuleTemplates = new LinkedHashMap<>(); - - for (IngestModuleTemplate template : ingestModuleTemplates) { - if (template.isDataSourceIngestModuleTemplate()) { - addModule(dataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); - } - if (template.isFileIngestModuleTemplate()) { - addModule(fileModuleTemplates, jythonFileModuleTemplates, template); - } - } - - /** - * Use the mappings and the ingest pipelines configuration to create - * ordered lists of ingest module templates for each ingest pipeline. - */ - IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates( - dataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - - List fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates( - fileModuleTemplates, jythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - - List secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates( - dataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); - - /** - * Add any module templates that were not specified in the pipelines - * configuration to an appropriate pipeline - either the first stage - * data source ingest pipeline or the file ingest pipeline. - */ - addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates, jythonDataSourceModuleTemplates); - addOrdered(fileIngestModuleTemplates, fileModuleTemplates, jythonFileModuleTemplates); - - /** - * Construct the data source ingest pipelines. - */ - this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - - /** - * Construct the file ingest pipelines, one per file ingest thread. - */ - try { - int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); - this.fileIngestPipelinesQueue.put(pipeline); - this.fileIngestPipelines.add(pipeline); - } - } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full queue. - * Blocking should actually never happen here, but reset the - * interrupted flag rather than just swallowing the exception. - */ - Thread.currentThread().interrupt(); - } - try { - SleuthkitCase skCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); - this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase); - this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); - } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest modules listing to case database", ex); - } - } - - private void addIngestModules(List templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException { - for (IngestModuleTemplate module : templates) { - ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber())); - } - } - - /** - * Uses an input collection of ingest module templates and a pipeline - * configuration, i.e., an ordered list of ingest module factory class - * names, to create an ordered output list of ingest module templates for an - * ingest pipeline. The ingest module templates are removed from the input - * collection as they are added to the output collection. - * - * @param ingestModuleTemplates A mapping of ingest module factory - * class names to ingest module - * templates. - * @param jythonIngestModuleTemplates A mapping of jython processed class - * names to jython ingest module - * templates. - * @param pipelineConfig An ordered list of ingest module - * factory class names representing an - * ingest pipeline. - * - * @return An ordered list of ingest module templates, i.e., an - * uninstantiated pipeline. - */ - private static List getConfiguredIngestModuleTemplates( - Map ingestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { - List templates = new ArrayList<>(); - for (String moduleClassName : pipelineConfig) { - if (ingestModuleTemplates != null && ingestModuleTemplates.containsKey(moduleClassName)) { - templates.add(ingestModuleTemplates.remove(moduleClassName)); - } else if (jythonIngestModuleTemplates != null && jythonIngestModuleTemplates.containsKey(moduleClassName)) { - templates.add(jythonIngestModuleTemplates.remove(moduleClassName)); - } - } - return templates; - } - - /** - * Gets the identifier of this job. - * - * @return The job identifier. - */ - long getId() { - return this.id; - } - - /** - * Get the ingest execution context identifier. - * - * @return The context string. - */ - String getExecutionContext() { - return this.settings.getExecutionContext(); - } - - /** - * Gets the data source to be ingested by this job. - * - * @return A Content object representing the data source. - */ - Content getDataSource() { - return this.dataSource; - } - - /** - * Queries whether or not unallocated space should be processed as part of - * this job. - * - * @return True or false. - */ - boolean shouldProcessUnallocatedSpace() { - return this.settings.getProcessUnallocatedSpace(); - } - - /** - * Gets the selected file ingest filter from settings. - * - * @return True or false. - */ - FilesSet getFileIngestFilter() { - return this.settings.getFileFilter(); - } - - /** - * Checks to see if this job has at least one ingest pipeline. - * - * @return True or false. - */ - boolean hasIngestPipeline() { - return this.hasFirstStageDataSourceIngestPipeline() - || this.hasFileIngestPipeline() - || this.hasSecondStageDataSourceIngestPipeline(); - } - - /** - * Checks to see if this job has a first stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasFirstStageDataSourceIngestPipeline() { - return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if this job has a second stage data source level ingest - * pipeline. - * - * @return True or false. - */ - private boolean hasSecondStageDataSourceIngestPipeline() { - return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); - } - - /** - * Checks to see if this job has a file level ingest pipeline. - * - * @return True or false. - */ - private boolean hasFileIngestPipeline() { - if (!this.fileIngestPipelines.isEmpty()) { - return !this.fileIngestPipelines.get(0).isEmpty(); - } - return false; - } - - /** - * Starts up the ingest pipelines for this job. - * - * @return A collection of ingest module startup errors, empty on success. - */ - List start() { - List errors = startUpIngestPipelines(); - if (errors.isEmpty()) { - try { - this.ingestJob = Case.getCurrentCaseThrows().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); - } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest job info to case database", ex); //NON-NLS - } - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - logInfoMessage("Starting first stage analysis"); //NON-NLS - this.startFirstStage(); - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - logInfoMessage("Starting second stage analysis"); //NON-NLS - this.startSecondStage(); - } - } - return errors; - } - - /** - * Starts up each of the ingest pipelines for this job to collect any file - * and data source level ingest modules errors that might occur. - * - * @return A collection of ingest module startup errors, empty on success. - */ - private List startUpIngestPipelines() { - List errors = new ArrayList<>(); - - /* - * Start the data-source-level ingest module pipelines. - */ - errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); - errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); - - /* - * If the data-source-level ingest pipelines were successfully started, - * start the Start the file-level ingest pipelines (one per file ingest - * thread). - */ - if (errors.isEmpty()) { - for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) { - errors.addAll(pipeline.startUp()); - if (!errors.isEmpty()) { - /* - * If there are start up errors, the ingest job will not - * proceed, so shut down any file ingest pipelines that did - * start up. - */ - while (!this.fileIngestPipelinesQueue.isEmpty()) { - FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll(); - if (startedPipeline.isRunning()) { - List shutDownErrors = startedPipeline.shutDown(); - if (!shutDownErrors.isEmpty()) { - /* - * The start up errors will ultimately be - * reported to the user for possible remedy, but - * the shut down errors are logged here. - */ - logIngestModuleErrors(shutDownErrors); - } - } - } - break; - } - } - } - - return errors; - } - - /** - * Starts the first stage of this job. - */ - private void startFirstStage() { - this.stage = DataSourceIngestJob.Stages.FIRST; - - if (this.hasFileIngestPipeline()) { - synchronized (this.fileIngestProgressLock) { - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); - } - } - - if (this.doUI) { - /** - * Start one or both of the first stage ingest progress bars. - */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - logInfoMessage("Scheduling first stage data source and file level analysis tasks"); //NON-NLS - DataSourceIngestJob.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - logInfoMessage("Scheduling first stage data source level analysis tasks"); //NON-NLS - DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - logInfoMessage("Scheduling file level analysis tasks, no first stage data source level analysis configured"); //NON-NLS - DataSourceIngestJob.taskScheduler.scheduleFileIngestTasks(this, this.files); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never get to check for completion of this stage of the job, so do - * it now. - */ - this.checkForStageCompleted(); - } - } - - /** - * Starts the second stage of this ingest job. - */ - private void startSecondStage() { - logInfoMessage("Starting second stage analysis"); //NON-NLS - this.stage = DataSourceIngestJob.Stages.SECOND; - if (this.doUI) { - this.startDataSourceIngestProgressBar(); - } - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; - } - logInfoMessage("Scheduling second stage data source level analysis tasks"); //NON-NLS - DataSourceIngestJob.taskScheduler.scheduleDataSourceIngestTask(this); - } - - /** - * Starts a data source level ingest progress bar for this job. - */ - private void startDataSourceIngestProgressBar() { - if (this.doUI) { - synchronized (this.dataSourceIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.dataSourceIngest.initialDisplayName", - this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. What remains to be done is to find out whether - // the user wants to cancel only the currently executing - // data source ingest module or the entire ingest job. - DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel(); - String dialogTitle = NbBundle.getMessage(DataSourceIngestJob.this.getClass(), "IngestJob.cancellationDialog.title"); - JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE); - if (panel.cancelAllDataSourceIngestModules()) { - DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - } else { - DataSourceIngestJob.this.cancelCurrentDataSourceIngestModule(); - } - return true; - } - }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - } - - /** - * Starts the file level ingest progress bar for this job. - */ - private void startFileIngestProgressBar() { - if (this.doUI) { - synchronized (this.fileIngestProgressLock) { - String displayName = NbBundle.getMessage(this.getClass(), - "IngestJob.progress.fileIngest.displayName", - this.dataSource.getName()); - this.fileIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - // If this method is called, the user has already pressed - // the cancel button on the progress bar and the OK button - // of a cancelation confirmation dialog supplied by - // NetBeans. - DataSourceIngestJob.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); - return true; - } - }); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); - } - } - } - - /** - * Checks to see if the ingest tasks for the current stage of this job are - * completed and does a stage transition if they are. - */ - private void checkForStageCompleted() { - synchronized (this.stageCompletionCheckLock) { - if (DataSourceIngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - switch (this.stage) { - case FIRST: - this.finishFirstStage(); - break; - case SECOND: - this.finish(); - break; - } - } - } - } - - /** - * Shuts down the first stage ingest pipelines and progress bars for this - * job and starts the second stage, if appropriate. - */ - private void finishFirstStage() { - logInfoMessage("Finished first stage analysis"); //NON-NLS - - // Shut down the file ingest pipelines. Note that no shut down is - // required for the data source ingest pipeline because data source - // ingest modules do not have a shutdown() method. - List errors = new ArrayList<>(); - while (!this.fileIngestPipelinesQueue.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); - if (pipeline.isRunning()) { - errors.addAll(pipeline.shutDown()); - } - } - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - if (this.doUI) { - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; - } - } - } - - /** - * Start the second stage, if appropriate. - */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } else { - this.finish(); - } - } - - /** - * Shuts down the ingest pipelines and progress bars for this job. - */ - private void finish() { - logInfoMessage("Finished analysis"); //NON-NLS - this.stage = DataSourceIngestJob.Stages.FINALIZATION; - - if (this.doUI) { - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } - if (ingestJob != null) { - if (this.cancelled) { - try { - ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } else { - try { - ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } - try { - this.ingestJob.setEndDateTime(new Date()); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); - } - } - this.parentJob.dataSourceJobFinished(this); - } - - /** - * Passes the data source for this job through the currently active data - * source level ingest pipeline. - * - * @param task A data source ingest task wrapping the data source. - */ - void process(DataSourceIngestTask task) { - try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - } - } - - if (this.doUI) { - /** - * Shut down the data source ingest progress bar right away. - * Data source-level processing is finished for this stage. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } - - } finally { - DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. - * - * @param task A file ingest task. - * - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or - * putting to the file ingest pipelines - * collection. - */ - void process(FileIngestTask task) throws InterruptedException { - try { - if (!this.isCancelled()) { - FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); - if (!pipeline.isEmpty()) { - AbstractFile file = task.getFile(); - - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.doUI) { - /** - * Update the file ingest progress bar. - */ - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); - } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); - } - this.filesInProgress.add(file.getName()); - } - } - - /** - * Run the file through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(pipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - - if (this.doUI && !this.cancelled) { - synchronized (this.fileIngestProgressLock) { - /** - * Update the file ingest progress bar again, in - * case the file was being displayed. - */ - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); - } else { - this.fileIngestProgress.progress(""); - } - } - } - } - this.fileIngestPipelinesQueue.put(pipeline); - } - } finally { - DataSourceIngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); - } - } - - /** - * Adds more files from the data source for this job to the job, e.g., adds - * extracted or carved files. Not currently supported for the second stage - * of the job. - * - * @param files A list of the files to add. - */ - void addFiles(List files) { - if (DataSourceIngestJob.Stages.FIRST == this.stage) { - DataSourceIngestJob.taskScheduler.fastTrackFileIngestTasks(this, files); - } else { - logErrorMessage(Level.SEVERE, "Adding files to job during second stage analysis not supported"); - } - - /** - * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a completion check would not be necessary, so this is a - * bit of defensive programming. - */ - this.checkForStageCompleted(); - } - - /** - * Updates the display name shown on the current data source level ingest - * progress bar for this job. - * - * @param displayName The new display name. - */ - void updateDataSourceIngestProgressBarDisplayName(String displayName) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); - } - } - } - - /** - * Switches the data source level ingest progress bar for this job to - * determinate mode. This should be called if the total work units to - * process the data source is known. - * - * @param workUnits Total number of work units for the processing of the - * data source. - */ - void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); - } - } - } - } - - /** - * Switches the data source level ingest progress bar for this job to - * indeterminate mode. This should be called if the total work units to - * process the data source is unknown. - */ - void switchDataSourceIngestProgressBarToIndeterminate() { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); - } - } - } - } - - /** - * Updates the data source level ingest progress bar for this job with the - * number of work units performed, if in the determinate mode. - * - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(int workUnits) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); - } - } - } - } - - /** - * Updates the data source level ingest progress for this job with a new - * task name, where the task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - */ - void advanceDataSourceIngestProgressBar(String currentTask) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); - } - } - } - } - - /** - * Updates the data source level ingest progress bar for this with a new - * task name and the number of work units performed, if in the determinate - * mode. The task name is the "subtitle" under the display name. - * - * @param currentTask The task name. - * @param workUnits Number of work units performed. - */ - void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { - if (this.doUI && !this.cancelled) { - synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); - } - } - } - - /** - * Queries whether or not a temporary cancellation of data source level - * ingest in order to stop the currently executing data source level ingest - * module is in effect for this job. - * - * @return True or false. - */ - boolean currentDataSourceIngestModuleIsCancelled() { - return this.currentDataSourceIngestModuleCancelled; - } - - /** - * Rescind a temporary cancellation of data source level ingest that was - * used to stop a single data source level ingest module for this job. - * - * @param moduleDisplayName The display name of the module that was stopped. - */ - void currentDataSourceIngestModuleCancellationCompleted(String moduleDisplayName) { - this.currentDataSourceIngestModuleCancelled = false; - this.cancelledDataSourceIngestModules.add(moduleDisplayName); - - if (this.doUI) { - /** - * A new progress bar must be created because the cancel button of - * the previously constructed component is disabled by NetBeans when - * the user selects the "OK" button of the cancellation confirmation - * dialog popped up by NetBeans when the progress bar cancel button - * is pressed. - */ - synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - this.startDataSourceIngestProgressBar(); - } - } - } - - /** - * Gets the currently running data source level ingest module for this job. - * - * @return The currently running module, may be null. - */ - DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() { - if (null != this.currentDataSourceIngestPipeline) { - return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule(); - } else { - return null; - } - } - - /** - * Requests a temporary cancellation of data source level ingest for this - * job in order to stop the currently executing data source ingest module. - */ - void cancelCurrentDataSourceIngestModule() { - this.currentDataSourceIngestModuleCancelled = true; - } - - /** - * Requests cancellation of ingest, i.e., a shutdown of the data source - * level and file level ingest pipelines. - * - * @param reason The cancellation reason. - */ - void cancel(IngestJob.CancellationReason reason) { - this.cancelled = true; - this.cancellationReason = reason; - DataSourceIngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); - - if (this.doUI) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != dataSourceIngestProgress) { - dataSourceIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", dataSource.getName())); - dataSourceIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); - } - } - - synchronized (this.fileIngestProgressLock) { - if (null != this.fileIngestProgress) { - this.fileIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName())); - this.fileIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); - } - } - } - } - - /** - * Set the current module name being run and the file name it is running on. - * To be used for more detailed cancelling. - * - * @param moduleName Name of module currently running. - * @param taskName Name of file the module is running on. - */ - void setCurrentFileIngestModule(String moduleName, String taskName) { - this.currentFileIngestModule = moduleName; - this.currentFileIngestTask = taskName; - } - - /** - * Queries whether or not cancellation, i.e., a shutdown of the data source - * level and file level ingest pipelines for this job, has been requested. - * - * @return True or false. - */ - boolean isCancelled() { - return this.cancelled; - } - - /** - * Gets the reason this job was cancelled. - * - * @return The cancellation reason, may be not cancelled. - */ - IngestJob.CancellationReason getCancellationReason() { - return this.cancellationReason; - } - - /** - * Writes an info message to the application log that includes the data - * source name, data source object id, and the job id. - * - * @param message The message. - */ - private void logInfoMessage(String message) { - logger.log(Level.INFO, String.format("%s (data source = %s, objId = %d, jobId = %d)", message, dataSource.getName(), dataSource.getId(), id)); //NON-NLS - } - - /** - * Writes an error message to the application log that includes the data - * source name, data source object id, and the job id. - * - * @param level The logging level for the message. - * @param message The message. - * @param throwable The throwable associated with the error. - */ - private void logErrorMessage(Level level, String message, Throwable throwable) { - logger.log(level, String.format("%s (data source = %s, objId = %d, jobId = %d)", message, dataSource.getName(), dataSource.getId(), id), throwable); //NON-NLS - } - - /** - * Writes an error message to the application log that includes the data - * source name, data source object id, and the job id. - * - * @param level The logging level for the message. - * @param message The message. - */ - private void logErrorMessage(Level level, String message) { - logger.log(level, String.format("%s (data source = %s, objId = %d, jobId = %d)", message, dataSource.getName(), dataSource.getId(), id)); //NON-NLS - } - - /** - * Write ingest module errors to the log. - * - * @param errors The errors. - */ - private void logIngestModuleErrors(List errors) { - for (IngestModuleError error : errors) { - logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis", error.getModuleDisplayName()), error.getThrowable()); //NON-NLS - } - } - - /** - * Gets a snapshot of this jobs state and performance. - * - * @return An ingest job statistics object. - */ - Snapshot getSnapshot(boolean getIngestTasksSnapshot) { - /** - * Determine whether file ingest is running at the time of this snapshot - * and determine the earliest file ingest level pipeline start time, if - * file ingest was started at all. - */ - boolean fileIngestRunning = false; - Date fileIngestStartTime = null; - - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { - if (pipeline.isRunning()) { - fileIngestRunning = true; - } - Date pipelineStartTime = pipeline.getStartTime(); - if (null != pipelineStartTime && (null == fileIngestStartTime || pipelineStartTime.before(fileIngestStartTime))) { - fileIngestStartTime = pipelineStartTime; - } - } - - long processedFilesCount = 0; - long estimatedFilesToProcessCount = 0; - long snapShotTime = new Date().getTime(); - IngestJobTasksSnapshot tasksSnapshot = null; - - if (getIngestTasksSnapshot) { - synchronized (fileIngestProgressLock) { - processedFilesCount = this.processedFiles; - estimatedFilesToProcessCount = this.estimatedFilesToProcess; - snapShotTime = new Date().getTime(); - } - tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(id); - - } - - return new Snapshot(this.dataSource.getName(), id, createTime, - getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime, - cancelled, cancellationReason, cancelledDataSourceIngestModules, - processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot); - } - - /** - * Stores basic diagnostic statistics for a data source ingest job. - */ - public static final class Snapshot implements Serializable { - - private static final long serialVersionUID = 1L; - - private final String dataSource; - private final long jobId; - private final long jobStartTime; - private final long snapShotTime; - transient private final PipelineModule dataSourceLevelIngestModule; - private final boolean fileIngestRunning; - private final Date fileIngestStartTime; - private final long processedFiles; - private final long estimatedFilesToProcess; - private final IngestJobTasksSnapshot tasksSnapshot; - transient private final boolean jobCancelled; - transient private final CancellationReason jobCancellationReason; - transient private final List cancelledDataSourceModules; - - /** - * Constructs an object to store basic diagnostic statistics for a data - * source ingest job. - */ - Snapshot(String dataSourceName, long jobId, long jobStartTime, PipelineModule dataSourceIngestModule, - boolean fileIngestRunning, Date fileIngestStartTime, - boolean jobCancelled, CancellationReason cancellationReason, List cancelledModules, - long processedFiles, long estimatedFilesToProcess, - long snapshotTime, IngestJobTasksSnapshot tasksSnapshot) { - this.dataSource = dataSourceName; - this.jobId = jobId; - this.jobStartTime = jobStartTime; - this.dataSourceLevelIngestModule = dataSourceIngestModule; - - this.fileIngestRunning = fileIngestRunning; - this.fileIngestStartTime = fileIngestStartTime; - this.jobCancelled = jobCancelled; - this.jobCancellationReason = cancellationReason; - this.cancelledDataSourceModules = cancelledModules; - - this.processedFiles = processedFiles; - this.estimatedFilesToProcess = estimatedFilesToProcess; - this.snapShotTime = snapshotTime; - this.tasksSnapshot = tasksSnapshot; - } - - /** - * Gets time these statistics were collected. - * - * @return The statistics collection time as number of milliseconds - * since January 1, 1970, 00:00:00 GMT. - */ - long getSnapshotTime() { - return snapShotTime; - } - - /** - * Gets the name of the data source associated with the ingest job that - * is the subject of this snapshot. - * - * @return A data source name string. - */ - String getDataSource() { - return dataSource; - } - - /** - * Gets the identifier of the ingest job that is the subject of this - * snapshot. - * - * @return The ingest job id. - */ - long getJobId() { - return this.jobId; - } - - /** - * Gets the time the ingest job was started. - * - * @return The start time as number of milliseconds since January 1, - * 1970, 00:00:00 GMT. - */ - long getJobStartTime() { - return jobStartTime; - } - - DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() { - return this.dataSourceLevelIngestModule; - } - - boolean getFileIngestIsRunning() { - return this.fileIngestRunning; - } - - Date getFileIngestStartTime() { - return this.fileIngestStartTime; - } - - /** - * Gets files per second throughput since the ingest job that is the - * subject of this snapshot started. - * - * @return Files processed per second (approximate). - */ - double getSpeed() { - return (double) processedFiles / ((snapShotTime - jobStartTime) / 1000); - } - - /** - * Gets the number of files processed for the job so far. - * - * @return The number of processed files. - */ - long getFilesProcessed() { - return processedFiles; - } - - /** - * Gets an estimate of the files that still need to be processed for - * this job. - * - * @return The estimate. - */ - long getFilesEstimated() { - return estimatedFilesToProcess; - } - - long getRootQueueSize() { - if (null == this.tasksSnapshot) { - return 0; - } - return this.tasksSnapshot.getRootQueueSize(); - } - - long getDirQueueSize() { - if (null == this.tasksSnapshot) { - return 0; - } - return this.tasksSnapshot.getDirectoryTasksQueueSize(); - } - - long getFileQueueSize() { - if (null == this.tasksSnapshot) { - return 0; - } - return this.tasksSnapshot.getFileQueueSize(); - } - - long getDsQueueSize() { - if (null == this.tasksSnapshot) { - return 0; - } - return this.tasksSnapshot.getDsQueueSize(); - } - - long getRunningListSize() { - if (null == this.tasksSnapshot) { - return 0; - } - return this.tasksSnapshot.getRunningListSize(); - } - - boolean isCancelled() { - return this.jobCancelled; - } - - /** - * Gets the reason this job was cancelled. - * - * @return The cancellation reason, may be not cancelled. - */ - IngestJob.CancellationReason getCancellationReason() { - return this.jobCancellationReason; - } - - /** - * Gets a list of the display names of any canceled data source level - * ingest modules - * - * @return A list of canceled data source level ingest module display - * names, possibly empty. - */ - List getCancelledDataSourceIngestModules() { - return Collections.unmodifiableList(this.cancelledDataSourceModules); - } - - } - -} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index 280613c2b3..a2bd23b692 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -23,10 +23,10 @@ package org.sleuthkit.autopsy.ingest; */ public class DataSourceIngestModuleProgress { - private final DataSourceIngestJob job; + private final IngestJobPipeline ingestJobPipeline; - DataSourceIngestModuleProgress(DataSourceIngestJob job) { - this.job = job; + DataSourceIngestModuleProgress(IngestJobPipeline pipeline) { + this.ingestJobPipeline = pipeline; } /** @@ -38,7 +38,7 @@ public class DataSourceIngestModuleProgress { * data source. */ public void switchToDeterminate(int workUnits) { - this.job.switchDataSourceIngestProgressBarToDeterminate(workUnits); + this.ingestJobPipeline.switchDataSourceIngestProgressBarToDeterminate(workUnits); } /** @@ -46,7 +46,7 @@ public class DataSourceIngestModuleProgress { * the total work units to process the data source is unknown. */ public void switchToIndeterminate() { - this.job.switchDataSourceIngestProgressBarToIndeterminate(); + this.ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate(); } /** @@ -56,7 +56,7 @@ public class DataSourceIngestModuleProgress { * @param workUnits Number of work units performed so far by the module. */ public void progress(int workUnits) { - this.job.advanceDataSourceIngestProgressBar("", workUnits); + this.ingestJobPipeline.advanceDataSourceIngestProgressBar("", workUnits); } /** @@ -65,7 +65,7 @@ public class DataSourceIngestModuleProgress { * @param message Message to display */ public void progress(String message) { - this.job.advanceDataSourceIngestProgressBar(message); + this.ingestJobPipeline.advanceDataSourceIngestProgressBar(message); } /** @@ -76,7 +76,7 @@ public class DataSourceIngestModuleProgress { * @param workUnits Number of work units performed so far by the module. */ public void progress(String currentTask, int workUnits) { - this.job.advanceDataSourceIngestProgressBar(currentTask, workUnits); + this.ingestJobPipeline.advanceDataSourceIngestProgressBar(currentTask, workUnits); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 12645df4e6..c22d63348c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -24,12 +24,11 @@ import java.util.List; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.datamodel.Content; /** - * This class manages a sequence of data source level ingest modules for a data - * source ingest job. It starts the modules, runs data sources through them, and + * This class manages a sequence of data source level ingest modules for an + * ingestJobPipeline. It starts the modules, runs data sources through them, and * shuts them down when data source level ingest is complete. *

* This class is thread-safe. @@ -38,7 +37,7 @@ final class DataSourceIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName()); - private final DataSourceIngestJob job; + private final IngestJobPipeline ingestJobPipeline; private final List modules = new ArrayList<>(); private volatile PipelineModule currentModule; @@ -47,13 +46,12 @@ final class DataSourceIngestPipeline { * modules. It starts the modules, runs data sources through them, and shuts * them down when data source level ingest is complete. * - * @param job The data source ingest job that owns this - * pipeline. + * @param ingestJobPipeline The ingestJobPipeline that owns this pipeline. * @param moduleTemplates Templates for the creating the ingest modules that * make up this pipeline. */ - DataSourceIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { - this.job = job; + DataSourceIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + this.ingestJobPipeline = ingestJobPipeline; for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName()); @@ -80,7 +78,7 @@ final class DataSourceIngestPipeline { List errors = new ArrayList<>(); for (PipelineModule module : modules) { try { - module.startUp(new IngestJobContext(this.job)); + module.startUp(new IngestJobContext(this.ingestJobPipeline)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } @@ -98,7 +96,7 @@ final class DataSourceIngestPipeline { */ synchronized List process(DataSourceIngestTask task) { List errors = new ArrayList<>(); - if (!this.job.isCancelled()) { + if (!this.ingestJobPipeline.isCancelled()) { Content dataSource = task.getDataSource(); for (PipelineModule module : modules) { try { @@ -106,19 +104,19 @@ final class DataSourceIngestPipeline { String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", module.getDisplayName(), dataSource.getName()); - this.job.updateDataSourceIngestProgressBarDisplayName(displayName); - this.job.switchDataSourceIngestProgressBarToIndeterminate(); + this.ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(displayName); + this.ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate(); DataSourceIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); - logger.log(Level.INFO, "{0} analysis of {1} (jobId={2}) starting", new Object[]{module.getDisplayName(), this.job.getDataSource().getName(), this.job.getId()}); //NON-NLS - module.process(dataSource, new DataSourceIngestModuleProgress(this.job)); - logger.log(Level.INFO, "{0} analysis of {1} (jobId={2}) finished", new Object[]{module.getDisplayName(), this.job.getDataSource().getName(), this.job.getId()}); //NON-NLS + logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) starting", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS + module.process(dataSource, new DataSourceIngestModuleProgress(this.ingestJobPipeline)); + logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) finished", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (this.job.isCancelled()) { + if (this.ingestJobPipeline.isCancelled()) { break; - } else if (this.job.currentDataSourceIngestModuleIsCancelled()) { - this.job.currentDataSourceIngestModuleCancellationCompleted(currentModule.getDisplayName()); + } else if (this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) { + this.ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(currentModule.getDisplayName()); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index bb83b3c63f..417b7bee96 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -20,13 +20,13 @@ package org.sleuthkit.autopsy.ingest; final class DataSourceIngestTask extends IngestTask { - DataSourceIngestTask(DataSourceIngestJob job) { - super(job); + DataSourceIngestTask(IngestJobPipeline ingestJobPipeline) { + super(ingestJobPipeline); } @Override void execute(long threadId) throws InterruptedException { super.setThreadId(threadId); - getIngestJob().process(this); + getIngestJobPipeline().process(this); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 44473e1b39..2f6604e415 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -24,15 +24,14 @@ import java.util.List; import java.util.logging.Level; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * This class manages a sequence of file level ingest modules for a data source - * ingest job. It starts the modules, runs files through them, and shuts them + * This class manages a sequence of file level ingest modules for an + * ingest job pipeline. It starts the modules, runs files through them, and shuts them * down when file level ingest is complete. *

* This class is thread-safe. @@ -40,7 +39,7 @@ import org.sleuthkit.datamodel.TskCoreException; final class FileIngestPipeline { private static final IngestManager ingestManager = IngestManager.getInstance(); - private final DataSourceIngestJob job; + private final IngestJobPipeline ingestJobPipeline; private final List modules = new ArrayList<>(); private Date startTime; private volatile boolean running; @@ -50,12 +49,12 @@ final class FileIngestPipeline { * modules. It starts the modules, runs files through them, and shuts them * down when file level ingest is complete. * - * @param job The data source ingest job that owns the pipeline. + * @param ingestJobPipeline The ingestJobPipeline that owns the pipeline. * @param moduleTemplates The ingest module templates that define the * pipeline. */ - FileIngestPipeline(DataSourceIngestJob job, List moduleTemplates) { - this.job = job; + FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + this.ingestJobPipeline = ingestJobPipeline; for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName()); @@ -103,7 +102,7 @@ final class FileIngestPipeline { List errors = new ArrayList<>(); for (PipelineModule module : this.modules) { try { - module.startUp(new IngestJobContext(this.job)); + module.startUp(new IngestJobContext(this.ingestJobPipeline)); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } @@ -120,22 +119,31 @@ final class FileIngestPipeline { */ synchronized List process(FileIngestTask task) { List errors = new ArrayList<>(); - if (!this.job.isCancelled()) { - AbstractFile file = task.getFile(); + if (!this.ingestJobPipeline.isCancelled()) { + AbstractFile file; + try { + file = task.getFile(); + } catch (TskCoreException ex) { + // In practice, this task would never have been enqueued since the file + // lookup would have failed there. + errors.add(new IngestModuleError("File Ingest Pipeline", ex)); // NON-NLS + FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task); + return errors; + } for (PipelineModule module : this.modules) { try { FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName()); - this.job.setCurrentFileIngestModule(module.getDisplayName(), task.getFile().getName()); + this.ingestJobPipeline.setCurrentFileIngestModule(module.getDisplayName(), task.getFile().getName()); module.process(file); } catch (Throwable ex) { // Catch-all exception firewall errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (this.job.isCancelled()) { + if (this.ingestJobPipeline.isCancelled()) { break; } } - if (!this.job.isCancelled()) { + if (!this.ingestJobPipeline.isCancelled()) { // Save any properties that have not already been saved to the database try{ file.save(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index e4a8209df0..3921915414 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -19,29 +19,45 @@ package org.sleuthkit.autopsy.ingest; import java.util.Objects; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; /** * Represents a single file analysis task, which is defined by a file to analyze * and the InjestJob/Pipeline to run it on. */ final class FileIngestTask extends IngestTask { + + private final long fileId; + private AbstractFile file = null; - private final AbstractFile file; - - FileIngestTask(DataSourceIngestJob job, AbstractFile file) { - super(job); + FileIngestTask(IngestJobPipeline ingestJobPipeline, AbstractFile file) { + super(ingestJobPipeline); this.file = file; + fileId = file.getId(); + } + + FileIngestTask(IngestJobPipeline ingestJobPipeline, long fileId) { + super(ingestJobPipeline); + this.fileId = fileId; + } + + long getFileId() { + return fileId; } - AbstractFile getFile() { + synchronized AbstractFile getFile() throws TskCoreException { + if (file == null) { + file = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(fileId); + } return file; } @Override void execute(long threadId) throws InterruptedException { super.setThreadId(threadId); - getIngestJob().process(this); + getIngestJobPipeline().process(this); } @Override @@ -53,22 +69,19 @@ final class FileIngestTask extends IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - DataSourceIngestJob job = getIngestJob(); - DataSourceIngestJob otherJob = other.getIngestJob(); - if (job != otherJob && (job == null || !job.equals(otherJob))) { + IngestJobPipeline thisPipeline = getIngestJobPipeline(); + IngestJobPipeline otherPipeline = other.getIngestJobPipeline(); + if (thisPipeline != otherPipeline && (thisPipeline == null || !thisPipeline.equals(otherPipeline))) { return false; } - if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { - return false; - } - return true; + return (this.fileId == other.fileId); } @Override public int hashCode() { int hash = 5; - hash = 47 * hash + Objects.hashCode(getIngestJob()); - hash = 47 * hash + Objects.hashCode(this.file); + hash = 47 * hash + Objects.hashCode(getIngestJobPipeline()); + hash = 47 * hash + Objects.hashCode(this.fileId); return hash; } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 720313ba15..1349956e23 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -27,10 +28,15 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; - +import java.util.logging.Level; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskDataException; /** * Analyzes one or more data sources using a set of ingest modules specified via @@ -60,30 +66,37 @@ public final class IngestJob { return displayName; } } - + + enum Mode { + BATCH, + STREAMING + } + + private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); private final static AtomicLong nextId = new AtomicLong(0L); private final long id; - private final Map dataSourceJobs; + private final List dataSources = new ArrayList<>(); + private final List files = new ArrayList<>(); + private final Mode ingestMode; + private final Map ingestJobPipelines; private final AtomicInteger incompleteJobsCount; + private final IngestJobSettings settings; private volatile CancellationReason cancellationReason; /** * Constructs an ingest job that analyzes one or more data sources using a - * set of ingest modules specified via ingest job settings. + * set of ingest modules specified via ingest settings. * * @param dataSources The data sources to be ingested. - * @param settings The ingest job settings. - * @param doUI Whether or not this job should use progress bars, - * message boxes for errors, etc. + * @param settings The ingest settings. */ - IngestJob(Collection dataSources, IngestJobSettings settings, boolean doUI) { + IngestJob(Collection dataSources, IngestJobSettings settings) { this.id = IngestJob.nextId.getAndIncrement(); - this.dataSourceJobs = new ConcurrentHashMap<>(); - for (Content dataSource : dataSources) { - DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, settings, doUI); - this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); - } - incompleteJobsCount = new AtomicInteger(dataSourceJobs.size()); + this.settings = settings; + this.ingestJobPipelines = new ConcurrentHashMap<>(); + this.ingestMode = Mode.BATCH; + this.dataSources.addAll(dataSources); + incompleteJobsCount = new AtomicInteger(dataSources.size()); cancellationReason = CancellationReason.NOT_CANCELLED; } @@ -92,18 +105,28 @@ public final class IngestJob { * ingest modules specified via ingest job settings. Either all of the files * in the data source or a given subset of the files will be analyzed. * - * @param dataSource The data source to be analyzed + * @param dataSource The data source to be analyzed. * @param files A subset of the files for the data source. * @param settings The ingest job settings. - * @param doUI Whether or not this job should use progress bars, - * message boxes for errors, etc. */ - IngestJob(Content dataSource, List files, IngestJobSettings settings, boolean doUI) { + IngestJob(Content dataSource, List files, IngestJobSettings settings) { + this(Arrays.asList(dataSource), settings); + this.files.addAll(files); + } + + /** + * Constructs an ingest job that analyzes one data source, possibly using + * an ingest stream. + * + * @param settings The ingest job settings. + */ + IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) { this.id = IngestJob.nextId.getAndIncrement(); - this.dataSourceJobs = new ConcurrentHashMap<>(); - DataSourceIngestJob dataSourceIngestJob = new DataSourceIngestJob(this, dataSource, files, settings, doUI); - this.dataSourceJobs.put(dataSourceIngestJob.getId(), dataSourceIngestJob); - incompleteJobsCount = new AtomicInteger(dataSourceJobs.size()); + this.ingestJobPipelines = new ConcurrentHashMap<>(); + this.dataSources.add(dataSource); + this.settings = settings; + this.ingestMode = ingestMode; + incompleteJobsCount = new AtomicInteger(1); cancellationReason = CancellationReason.NOT_CANCELLED; } @@ -124,18 +147,35 @@ public final class IngestJob { * @return True or false. */ boolean hasIngestPipeline() { - /** - * TODO: This could actually be done more simply by adding a method to - * the IngestJobSettings to check for at least one enabled ingest module - * template. The test could then be done in the ingest manager before - * even constructing an ingest job. - */ - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - if (dataSourceJob.hasIngestPipeline()) { - return true; - } + return (!settings.getEnabledIngestModuleTemplates().isEmpty()); + } + + /** + * Add a set of files (by object ID) to be ingested. + * + * @param fileObjIds the list of file IDs + */ + void addStreamingIngestFiles(List fileObjIds) { + if (ingestJobPipelines.isEmpty()) { + logger.log(Level.SEVERE, "Attempted to add streaming ingest files with no IngestJobPipeline"); + return; } - return false; + // Streaming ingest jobs will only have one data source + IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); + streamingIngestPipeline.addStreamingIngestFiles(fileObjIds); + } + + /** + * Start data source processing for streaming ingest. + */ + void processStreamingIngestDataSource() { + if (ingestJobPipelines.isEmpty()) { + logger.log(Level.SEVERE, "Attempted to start data source ingest with no IngestJobPipeline"); + return; + } + // Streaming ingest jobs will only have one data source + IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); + streamingIngestPipeline.processStreamingIngestDataSource(); } /** @@ -145,17 +185,32 @@ public final class IngestJob { * @return A collection of ingest module start up errors, empty on success. */ List start() { + /* - * Try to start each data source ingest job. Note that there is a not - * unwarranted assumption here that if there is going to be a module - * startup failure, it will be for the first data source ingest job. + * Set up the pipeline(s) + */ + if (files.isEmpty()) { + for (Content dataSource : dataSources) { + IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSource, settings); + this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); + } + } else { + IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSources.get(0), files, settings); + this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); + } + incompleteJobsCount.set(ingestJobPipelines.size()); + + /* + * Try to start each data source ingest job. Note that there is an + * assumption here that if there is going to be a module + * startup failure, it will be for the first ingest job pipeline. * * TODO (RC): Consider separating module start up from pipeline startup * so that no processing is done if this assumption is false. */ List errors = new ArrayList<>(); - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { - errors.addAll(dataSourceJob.start()); + for (IngestJobPipeline ingestJobPipeline : this.ingestJobPipelines.values()) { + errors.addAll(ingestJobPipeline.start()); if (errors.isEmpty() == false) { break; } @@ -165,7 +220,7 @@ public final class IngestJob { * Handle start up success or failure. */ if (errors.isEmpty()) { - for (DataSourceIngestJob dataSourceJob : this.dataSourceJobs.values()) { + for (IngestJobPipeline dataSourceJob : this.ingestJobPipelines.values()) { IngestManager.getInstance().fireDataSourceAnalysisStarted(id, dataSourceJob.getId(), dataSourceJob.getDataSource()); } } else { @@ -174,6 +229,15 @@ public final class IngestJob { return errors; } + + /** + * Get the ingest mode for this job (batch or streaming). + * + * @return the ingest mode. + */ + Mode getIngestMode() { + return ingestMode; + } /** * Gets a snapshot of the progress of this ingest job. @@ -187,6 +251,8 @@ public final class IngestJob { /** * Gets a snapshot of the progress of this ingest job. * + * @param getIngestTasksSnapshot + * * @return The snapshot. */ public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) { @@ -199,9 +265,9 @@ public final class IngestJob { * * @return A list of data source ingest job progress snapshots. */ - List getDataSourceIngestJobSnapshots() { - List snapshots = new ArrayList<>(); - this.dataSourceJobs.values().stream().forEach((dataSourceJob) -> { + List getDataSourceIngestJobSnapshots() { + List snapshots = new ArrayList<>(); + this.ingestJobPipelines.values().stream().forEach((dataSourceJob) -> { snapshots.add(dataSourceJob.getSnapshot(true)); }); return snapshots; @@ -230,7 +296,7 @@ public final class IngestJob { */ public void cancel(CancellationReason reason) { this.cancellationReason = reason; - this.dataSourceJobs.values().stream().forEach((job) -> { + this.ingestJobPipelines.values().stream().forEach((job) -> { job.cancel(reason); }); } @@ -255,17 +321,17 @@ public final class IngestJob { } /** - * Provides a callback for completed data source ingest jobs, allowing this + * Provides a callback for completed ingest job pipeline, allowing this * ingest job to notify the ingest manager when it is complete. * - * @param job A completed data source ingest job. + * @param ingestJobPipeline A completed ingestJobPipeline. */ - void dataSourceJobFinished(DataSourceIngestJob job) { + void ingestJobPipelineFinished(IngestJobPipeline ingestJobPipeline) { IngestManager ingestManager = IngestManager.getInstance(); - if (!job.isCancelled()) { - ingestManager.fireDataSourceAnalysisCompleted(id, job.getId(), job.getDataSource()); + if (!ingestJobPipeline.isCancelled()) { + ingestManager.fireDataSourceAnalysisCompleted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); } else { - IngestManager.getInstance().fireDataSourceAnalysisCancelled(id, job.getId(), job.getDataSource()); + IngestManager.getInstance().fireDataSourceAnalysisCancelled(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); } if (incompleteJobsCount.decrementAndGet() == 0) { ingestManager.finishIngestJob(this); @@ -290,9 +356,9 @@ public final class IngestJob { */ public final class DataSourceProcessingSnapshot { - private final DataSourceIngestJob.Snapshot snapshot; + private final Snapshot snapshot; - private DataSourceProcessingSnapshot(DataSourceIngestJob.Snapshot snapshot) { + private DataSourceProcessingSnapshot(Snapshot snapshot) { this.snapshot = snapshot; } @@ -346,13 +412,13 @@ public final class IngestJob { fileIngestRunning = false; fileIngestStartTime = null; dataSourceProcessingSnapshots = new ArrayList<>(); - for (DataSourceIngestJob dataSourceJob : dataSourceJobs.values()) { - DataSourceIngestJob.Snapshot snapshot = dataSourceJob.getSnapshot(getIngestTasksSnapshot); + for (IngestJobPipeline pipeline : ingestJobPipelines.values()) { + Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot); dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot)); if (null == dataSourceModule) { DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule(); if (null != module) { - dataSourceModule = new DataSourceIngestModuleHandle(dataSourceJobs.get(snapshot.getJobId()), module); + dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module); } } if (snapshot.getFileIngestIsRunning()) { @@ -433,7 +499,7 @@ public final class IngestJob { */ public static class DataSourceIngestModuleHandle { - private final DataSourceIngestJob job; + private final IngestJobPipeline ingestJobPipeline; private final DataSourceIngestPipeline.PipelineModule module; private final boolean cancelled; @@ -442,14 +508,13 @@ public final class IngestJob { * used to get basic information about the module and to request * cancellation of the module. * - * @param job The data source ingest job that owns the data source - * level ingest module. + * @param ingestJobPipeline The ingestJobPipeline that owns the data source level ingest module. * @param module The data source level ingest module. */ - private DataSourceIngestModuleHandle(DataSourceIngestJob job, DataSourceIngestPipeline.PipelineModule module) { - this.job = job; + private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.PipelineModule module) { + this.ingestJobPipeline = ingestJobPipeline; this.module = module; - this.cancelled = job.currentDataSourceIngestModuleIsCancelled(); + this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled(); } /** @@ -499,8 +564,8 @@ public final class IngestJob { * modules participating in this workaround will need to consult the * cancelled flag in the adapters. */ - if (this.job.getCurrentDataSourceIngestModule() == this.module) { - this.job.cancelCurrentDataSourceIngestModule(); + if (this.ingestJobPipeline.getCurrentDataSourceIngestModule() == this.module) { + this.ingestJobPipeline.cancelCurrentDataSourceIngestModule(); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 74da1d814b..697e512ad8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -28,10 +28,10 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJobContext { - private final DataSourceIngestJob ingestJob; + private final IngestJobPipeline ingestJobPipeline; - IngestJobContext(DataSourceIngestJob ingestJob) { - this.ingestJob = ingestJob; + IngestJobContext(IngestJobPipeline ingestJobPipeline) { + this.ingestJobPipeline = ingestJobPipeline; } /** @@ -40,7 +40,7 @@ public final class IngestJobContext { * @return The context string. */ public String getExecutionContext() { - return this.ingestJob.getExecutionContext(); + return this.ingestJobPipeline.getExecutionContext(); } /** @@ -49,7 +49,7 @@ public final class IngestJobContext { * @return The data source. */ public Content getDataSource() { - return this.ingestJob.getDataSource(); + return this.ingestJobPipeline.getDataSource(); } /** @@ -58,7 +58,7 @@ public final class IngestJobContext { * @return The ingest job identifier. */ public long getJobId() { - return this.ingestJob.getId(); + return this.ingestJobPipeline.getId(); } /** @@ -83,7 +83,7 @@ public final class IngestJobContext { * @return True or false. */ public boolean dataSourceIngestIsCancelled() { - return this.ingestJob.currentDataSourceIngestModuleIsCancelled() || this.ingestJob.isCancelled(); + return this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled() || this.ingestJobPipeline.isCancelled(); } /** @@ -94,7 +94,7 @@ public final class IngestJobContext { * @return True or false. */ public boolean fileIngestIsCancelled() { - return this.ingestJob.isCancelled(); + return this.ingestJobPipeline.isCancelled(); } /** @@ -104,7 +104,7 @@ public final class IngestJobContext { * @return True or false. */ public boolean processingUnallocatedSpace() { - return this.ingestJob.shouldProcessUnallocatedSpace(); + return this.ingestJobPipeline.shouldProcessUnallocatedSpace(); } /** @@ -127,7 +127,7 @@ public final class IngestJobContext { * @param files The files to be added. */ public void addFilesToJob(List files) { - this.ingestJob.addFiles(files); + this.ingestJobPipeline.addFiles(files); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index e345db3d65..e113c439d7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -67,6 +67,8 @@ import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisStartedEvent; import org.sleuthkit.autopsy.ingest.events.FileAnalyzedEvent; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.TskCoreException; /** * Manages the creation and execution of ingest jobs, i.e., the processing of @@ -285,6 +287,20 @@ public class IngestManager implements IngestProgressSnapshotProvider { caseIsOpen = false; clearIngestMessageBox(); } + + /** + * Creates an ingest stream from the given ingest settings for a data source. + * + * @param dataSource The data source + * @param settings The ingest job settings. + * + * @return The newly created ingest stream + */ + public IngestStream openIngestStream(DataSource dataSource, IngestJobSettings settings) { + IngestJob job = new IngestJob(dataSource, IngestJob.Mode.STREAMING, settings); + return new IngestJobInputStream(job); + } + /** * Gets the number of file ingest threads the ingest manager is using to do @@ -304,7 +320,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { */ public void queueIngestJob(Collection dataSources, IngestJobSettings settings) { if (caseIsOpen) { - IngestJob job = new IngestJob(dataSources, settings, RuntimeProperties.runningWithGUI()); + IngestJob job = new IngestJob(dataSources, settings); if (job.hasIngestPipeline()) { long taskId = nextIngestManagerTaskId.incrementAndGet(); Future task = startIngestJobsExecutor.submit(new StartIngestJobTask(taskId, job)); @@ -323,7 +339,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { */ public void queueIngestJob(Content dataSource, List files, IngestJobSettings settings) { if (caseIsOpen) { - IngestJob job = new IngestJob(dataSource, files, settings, RuntimeProperties.runningWithGUI()); + IngestJob job = new IngestJob(dataSource, files, settings); if (job.hasIngestPipeline()) { long taskId = nextIngestManagerTaskId.incrementAndGet(); Future task = startIngestJobsExecutor.submit(new StartIngestJobTask(taskId, job)); @@ -333,7 +349,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { } /** - * Immdiately starts an ingest job for one or more data sources. + * Immediately starts an ingest job for one or more data sources. * * @param dataSources The data sources to process. * @param settings The settings for the ingest job. @@ -343,7 +359,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { */ public IngestJobStartResult beginIngestJob(Collection dataSources, IngestJobSettings settings) { if (caseIsOpen) { - IngestJob job = new IngestJob(dataSources, settings, RuntimeProperties.runningWithGUI()); + IngestJob job = new IngestJob(dataSources, settings); if (job.hasIngestPipeline()) { return startIngestJob(job); } @@ -366,7 +382,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { "IngestManager.startupErr.dlgSolution=Please disable the failed modules or fix the errors before restarting ingest.", "IngestManager.startupErr.dlgErrorList=Errors:" }) - private IngestJobStartResult startIngestJob(IngestJob job) { + IngestJobStartResult startIngestJob(IngestJob job) { List errors = null; Case openCase; try { @@ -730,7 +746,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { * the task. */ void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) { - ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource())); + ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), ingestModuleDisplayName, task.getDataSource())); } /** @@ -746,7 +762,15 @@ public class IngestManager implements IngestProgressSnapshotProvider { */ void setIngestTaskProgress(FileIngestTask task, String ingestModuleDisplayName) { IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId()); - IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile()); + IngestThreadActivitySnapshot newSnap; + try { + AbstractFile file = task.getFile(); + newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile()); + } catch (TskCoreException ex) { + // In practice, this task would never have been enqueued or processed since the file + // lookup would have failed. + newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJobPipeline().getId(), ingestModuleDisplayName, task.getDataSource()); + } ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap); incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime()); } @@ -828,8 +852,8 @@ public class IngestManager implements IngestProgressSnapshotProvider { * @return A list of ingest job state snapshots. */ @Override - public List getIngestJobSnapshots() { - List snapShots = new ArrayList<>(); + public List getIngestJobSnapshots() { + List snapShots = new ArrayList<>(); synchronized (ingestJobsById) { ingestJobsById.values().forEach((job) -> { snapShots.addAll(job.getDataSourceIngestJobSnapshots()); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 44c508c75c..67e8ff55e8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -183,7 +183,7 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { "IngestJobTableModel.colName.rootQueued"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dsQueued")}; - private List jobSnapshots; + private List jobSnapshots; private IngestJobTableModel() { refresh(); @@ -211,7 +211,7 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { @Override public Object getValueAt(int rowIndex, int columnIndex) { - DataSourceIngestJob.Snapshot snapShot = jobSnapshots.get(rowIndex); + Snapshot snapShot = jobSnapshots.get(rowIndex); Object cellValue; switch (columnIndex) { case 0: diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotProvider.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotProvider.java index 511939e1f9..fc085615c7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotProvider.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotProvider.java @@ -38,7 +38,7 @@ public interface IngestProgressSnapshotProvider { * * @return A list of ingest job snapshots. */ - List getIngestJobSnapshots(); + List getIngestJobSnapshots(); /** * Gets the cumulative run times for the ingest module. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index de5e750244..ebfabf2ab2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -23,20 +23,20 @@ import org.sleuthkit.datamodel.Content; abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; - private final DataSourceIngestJob job; + private final IngestJobPipeline ingestJobPipeline; private long threadId; - IngestTask(DataSourceIngestJob job) { - this.job = job; + IngestTask(IngestJobPipeline ingestJobPipeline) { + this.ingestJobPipeline = ingestJobPipeline; threadId = NOT_SET; } - DataSourceIngestJob getIngestJob() { - return job; + IngestJobPipeline getIngestJobPipeline() { + return ingestJobPipeline; } Content getDataSource() { - return getIngestJob().getDataSource(); + return getIngestJobPipeline().getDataSource(); } long getThreadId() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 57da5330c0..4a3aef2d72 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -27,6 +27,7 @@ import java.util.Deque; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Queue; import java.util.TreeSet; import java.util.concurrent.BlockingDeque; import java.util.concurrent.LinkedBlockingDeque; @@ -58,6 +59,8 @@ final class IngestTasksScheduler { private final TreeSet rootFileTaskQueue; @GuardedBy("this") private final Deque pendingFileTaskQueue; + @GuardedBy("this") + private final Queue streamedTasksQueue; private final IngestTaskTrackingQueue fileIngestThreadsQueue; /** @@ -82,6 +85,7 @@ final class IngestTasksScheduler { this.rootFileTaskQueue = new TreeSet<>(new RootDirectoryTaskComparator()); this.pendingFileTaskQueue = new LinkedList<>(); this.fileIngestThreadsQueue = new IngestTaskTrackingQueue(); + this.streamedTasksQueue = new LinkedList<>(); } /** @@ -105,38 +109,38 @@ final class IngestTasksScheduler { } /** - * Schedules a data source level ingest task and zero to many file level - * ingest tasks for a data source ingest job. + * Schedules a data source level ingest task and zero to many file level + * ingest tasks for an ingest job pipeline. * - * @param job The data source ingest job. + * @param ingestJobPipeline The ingest job pipeline. */ - synchronized void scheduleIngestTasks(DataSourceIngestJob job) { - if (!job.isCancelled()) { + synchronized void scheduleIngestTasks(IngestJobPipeline ingestJobPipeline) { + if (!ingestJobPipeline.isCancelled()) { /* * Scheduling of both the data source ingest task and the initial - * file ingest tasks for a job must be an atomic operation. + * file ingest tasks for an ingestJobPipeline must be an atomic operation. * Otherwise, the data source task might be completed before the * file tasks are scheduled, resulting in a potential false positive * when another thread checks whether or not all the tasks for the - * job are completed. + * ingestJobPipeline are completed. */ - this.scheduleDataSourceIngestTask(job); - this.scheduleFileIngestTasks(job, Collections.emptyList()); + this.scheduleDataSourceIngestTask(ingestJobPipeline); + this.scheduleFileIngestTasks(ingestJobPipeline, Collections.emptyList()); } } /** - * Schedules a data source level ingest task for a data source ingest job. + * Schedules a data source level ingest task for an ingest job pipeline. * - * @param job The data source ingest job. + * @param ingestJobPipeline The ingest job pipeline. */ - synchronized void scheduleDataSourceIngestTask(DataSourceIngestJob job) { - if (!job.isCancelled()) { - DataSourceIngestTask task = new DataSourceIngestTask(job); + synchronized void scheduleDataSourceIngestTask(IngestJobPipeline ingestJobPipeline) { + if (!ingestJobPipeline.isCancelled()) { + DataSourceIngestTask task = new DataSourceIngestTask(ingestJobPipeline); try { this.dataSourceIngestThreadQueue.putLast(task); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (jobId={%d)", job.getId()), ex); + IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (jobId={%d)", ingestJobPipeline.getId()), ex); Thread.currentThread().interrupt(); } } @@ -144,40 +148,59 @@ final class IngestTasksScheduler { /** * Schedules file tasks for either all the files or a given subset of the - * files for a data source source ingest job. + * files for an ingest job pipeline. * - * @param job The data source ingest job. + * @param ingestJobPipeline The ingest job pipeline. * @param files A subset of the files for the data source; if empty, then * file tasks for all files in the data source are scheduled. */ - synchronized void scheduleFileIngestTasks(DataSourceIngestJob job, Collection files) { - if (!job.isCancelled()) { + synchronized void scheduleFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { + if (!ingestJobPipeline.isCancelled()) { Collection candidateFiles; if (files.isEmpty()) { - candidateFiles = getTopLevelFiles(job.getDataSource()); + candidateFiles = getTopLevelFiles(ingestJobPipeline.getDataSource()); } else { candidateFiles = files; } for (AbstractFile file : candidateFiles) { - FileIngestTask task = new FileIngestTask(job, file); + FileIngestTask task = new FileIngestTask(ingestJobPipeline, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { this.rootFileTaskQueue.add(task); } } - shuffleFileTaskQueues(); + refillIngestThreadQueue(); } } + + /** + * Schedules file tasks for the given list of file IDs. + * + * @param ingestJobPipeline The ingest job pipeline. + * @param files A subset of the files for the data source; if empty, then + * file tasks for all files in the data source are scheduled. + */ + synchronized void scheduleStreamedFileIngestTasks(IngestJobPipeline ingestJobPipeline, List fileIds) { + if (!ingestJobPipeline.isCancelled()) { + for (long id : fileIds) { + // Create the file ingest task. Note that we do not do the shouldEnqueueFileTask() + // check here in order to delay loading the AbstractFile object. + FileIngestTask task = new FileIngestTask(ingestJobPipeline, id); + this.streamedTasksQueue.add(task); + } + refillIngestThreadQueue(); + } + } /** - * Schedules file level ingest tasks for a given set of files for a data - * source ingest job by adding them directly to the front of the file tasks + * Schedules file level ingest tasks for a given set of files for an ingest + * job pipeline by adding them directly to the front of the file tasks * queue for the ingest manager's file ingest threads. * - * @param job The data source ingest job. + * @param ingestJobPipeline The ingestJobPipeline. * @param files A set of files for the data source. */ - synchronized void fastTrackFileIngestTasks(DataSourceIngestJob job, Collection files) { - if (!job.isCancelled()) { + synchronized void fastTrackFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { + if (!ingestJobPipeline.isCancelled()) { /* * Put the files directly into the queue for the file ingest * threads, if they pass the file filter for the job. The files are @@ -187,12 +210,12 @@ final class IngestTasksScheduler { * already in progress. */ for (AbstractFile file : files) { - FileIngestTask fileTask = new FileIngestTask(job, file); + FileIngestTask fileTask = new FileIngestTask(ingestJobPipeline, file); if (shouldEnqueueFileTask(fileTask)) { try { this.fileIngestThreadsQueue.putFirst(fileTask); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while scheduling file level ingest tasks (jobId={%d)", job.getId()), ex); + IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while scheduling file level ingest tasks (jobId={%d)", ingestJobPipeline.getId()), ex); Thread.currentThread().interrupt(); return; } @@ -219,34 +242,36 @@ final class IngestTasksScheduler { */ synchronized void notifyTaskCompleted(FileIngestTask task) { this.fileIngestThreadsQueue.taskCompleted(task); - shuffleFileTaskQueues(); + refillIngestThreadQueue(); } /** * Queries the task scheduler to determine whether or not all of the ingest - * tasks for a data source ingest job have been completed. + * tasks for an ingest job pipeline have been completed. * - * @param job The data source ingest job. + * @param ingestJobPipeline The ingestJobPipeline. * * @return True or false. */ - synchronized boolean tasksForJobAreCompleted(DataSourceIngestJob job) { - long jobId = job.getId(); + synchronized boolean currentTasksAreCompleted(IngestJobPipeline ingestJobPipeline) { + long jobId = ingestJobPipeline.getId(); + return !(this.dataSourceIngestThreadQueue.hasTasksForJob(jobId) || hasTasksForJob(this.rootFileTaskQueue, jobId) || hasTasksForJob(this.pendingFileTaskQueue, jobId) + || hasTasksForJob(this.streamedTasksQueue, jobId) || this.fileIngestThreadsQueue.hasTasksForJob(jobId)); } /** - * Clears the "upstream" task scheduling queues for a data source ingest - * job, but does nothing about tasks that have already been moved into the + * Clears the "upstream" task scheduling queues for an ingest pipeline, + * but does nothing about tasks that have already been moved into the * queue that is consumed by the file ingest threads. * - * @param job The data source ingest job. + * @param ingestJobPipeline The ingestJobPipeline. */ - synchronized void cancelPendingTasksForIngestJob(DataSourceIngestJob job) { - long jobId = job.getId(); + synchronized void cancelPendingTasksForIngestJob(IngestJobPipeline ingestJobPipeline) { + long jobId = ingestJobPipeline.getId(); IngestTasksScheduler.removeTasksForJob(this.rootFileTaskQueue, jobId); IngestTasksScheduler.removeTasksForJob(this.pendingFileTaskQueue, jobId); } @@ -291,6 +316,47 @@ final class IngestTasksScheduler { } return topLevelFiles; } + + /** + * Schedules file ingest tasks for the ingest manager's file ingest threads. + * Files from streaming ingest will be prioritized. + */ + synchronized private void refillIngestThreadQueue() { + try { + takeFromStreamingTaskQueue(); + takeFromBatchTasksQueues(); + } catch (InterruptedException ex) { + IngestTasksScheduler.logger.log(Level.INFO, "Ingest tasks scheduler interrupted while blocked adding a task to the file level ingest task queue", ex); + Thread.currentThread().interrupt(); + } + } + + /** + * Move tasks from the streamedTasksQueue into the fileIngestThreadsQueue. + * Will attempt to move as many tasks as there are ingest threads. + */ + synchronized private void takeFromStreamingTaskQueue() throws InterruptedException { + /* + * Schedule files from the streamedTasksQueue + */ + while (fileIngestThreadsQueue.isEmpty()) { + /* + * We will attempt to schedule as many tasks as there are ingest queues. + */ + int taskCount = 0; + while (taskCount < IngestManager.getInstance().getNumberOfFileIngestThreads()) { + final FileIngestTask streamingTask = streamedTasksQueue.poll(); + if (streamingTask == null) { + return; // No streaming tasks are queued right now + } + + if (shouldEnqueueFileTask(streamingTask)) { + fileIngestThreadsQueue.putLast(streamingTask); + taskCount++; + } + } + } + } /** * Schedules file ingest tasks for the ingest manager's file ingest threads @@ -322,8 +388,9 @@ final class IngestTasksScheduler { * during ingest. The reason for the LIFO additions is to give priority to * files derived from prioritized files. */ - synchronized private void shuffleFileTaskQueues() { - while (this.fileIngestThreadsQueue.isEmpty()) { + synchronized private void takeFromBatchTasksQueues() throws InterruptedException { + + while (this.fileIngestThreadsQueue.isEmpty()) { /* * If the pending file task queue is empty, move the highest * priority root file task, if there is one, into it. @@ -345,17 +412,11 @@ final class IngestTasksScheduler { return; } if (shouldEnqueueFileTask(pendingTask)) { - try { - /* - * The task is added to the queue for the ingest threads - * AFTER the higher priority tasks that preceded it. - */ - this.fileIngestThreadsQueue.putLast(pendingTask); - } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, "Ingest tasks scheduler interrupted while blocked adding a task to the file level ingest task queue", ex); - Thread.currentThread().interrupt(); - return; - } + /* + * The task is added to the queue for the ingest threads + * AFTER the higher priority tasks that preceded it. + */ + this.fileIngestThreadsQueue.putLast(pendingTask); } /* @@ -365,27 +426,27 @@ final class IngestTasksScheduler { * own, or into the queue for the file ingest threads, if it passes * the filter for the job. */ - final AbstractFile file = pendingTask.getFile(); + AbstractFile file = null; try { + file = pendingTask.getFile(); for (Content child : file.getChildren()) { if (child instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) child; - FileIngestTask childTask = new FileIngestTask(pendingTask.getIngestJob(), childFile); + FileIngestTask childTask = new FileIngestTask(pendingTask.getIngestJobPipeline(), childFile); if (childFile.hasChildren()) { this.pendingFileTaskQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - try { - this.fileIngestThreadsQueue.putLast(childTask); - } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, "Ingest tasks scheduler interrupted while blocked adding a task to the file level ingest task queue", ex); - Thread.currentThread().interrupt(); - return; - } + this.fileIngestThreadsQueue.putLast(childTask); } } } } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", file.getName(), file.getId()), ex); //NON-NLS + if (file != null) { + logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", file.getName(), file.getId()), ex); //NON-NLS + } else { + // In practice, the task would have already returned false from the call to shouldEnqueueFileTask() + logger.log(Level.SEVERE, "Error loading file with object ID {0}", pendingTask.getFileId()); + } } } } @@ -400,7 +461,13 @@ final class IngestTasksScheduler { * @return True or false. */ private static boolean shouldEnqueueFileTask(final FileIngestTask task) { - final AbstractFile file = task.getFile(); + AbstractFile file; + try { + file = task.getFile(); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error loading file with ID {0}", task.getFileId()); + return false; + } // Skip the task if the file is actually the pseudo-file for the parent // or current directory. @@ -483,7 +550,12 @@ final class IngestTasksScheduler { * @return True or false. */ private static boolean shouldBeCarved(final FileIngestTask task) { - return task.getIngestJob().shouldProcessUnallocatedSpace() && task.getFile().getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS); + try { + AbstractFile file = task.getFile(); + return task.getIngestJobPipeline().shouldProcessUnallocatedSpace() && file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS); + } catch (TskCoreException ex) { + return false; + } } /** @@ -495,7 +567,12 @@ final class IngestTasksScheduler { * @return True or false. */ private static boolean fileAcceptedByFilter(final FileIngestTask task) { - return !(task.getIngestJob().getFileIngestFilter().fileIsMemberOf(task.getFile()) == null); + try { + AbstractFile file = task.getFile(); + return !(task.getIngestJobPipeline().getFileIngestFilter().fileIsMemberOf(file) == null); + } catch (TskCoreException ex) { + return false; + } } /** @@ -509,7 +586,7 @@ final class IngestTasksScheduler { */ synchronized private static boolean hasTasksForJob(Collection tasks, long jobId) { for (IngestTask task : tasks) { - if (task.getIngestJob().getId() == jobId) { + if (task.getIngestJobPipeline().getId() == jobId) { return true; } } @@ -527,7 +604,7 @@ final class IngestTasksScheduler { Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); - if (task.getIngestJob().getId() == jobId) { + if (task.getIngestJobPipeline().getId() == jobId) { iterator.remove(); } } @@ -544,7 +621,7 @@ final class IngestTasksScheduler { private static int countTasksForJob(Collection queue, long jobId) { int count = 0; for (IngestTask task : queue) { - if (task.getIngestJob().getId() == jobId) { + if (task.getIngestJobPipeline().getId() == jobId) { count++; } } @@ -575,10 +652,36 @@ final class IngestTasksScheduler { @Override public int compare(FileIngestTask q1, FileIngestTask q2) { - AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile()); - AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile()); + // In practice the case where one or both calls to getFile() fails + // should never occur since such tasks would not be added to the queue. + AbstractFile file1 = null; + AbstractFile file2 = null; + try { + file1 = q1.getFile(); + } catch (TskCoreException ex) { + // Do nothing - the exception has been logged elsewhere + } + + try { + file2 = q2.getFile(); + } catch (TskCoreException ex) { + // Do nothing - the exception has been logged elsewhere + } + + if (file1 == null) { + if (file2 == null) { + return (int) (q2.getFileId() - q1.getFileId()); + } else { + return 1; + } + } else if (file2 == null) { + return -1; + } + + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(file1); + AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(file2); if (p1 == p2) { - return (int) (q2.getFile().getId() - q1.getFile().getId()); + return (int) (file2.getId() - file1.getId()); } else { return p2.ordinal() - p1.ordinal(); } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java index bedf15bdae..f5a2f2bd8c 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java @@ -29,6 +29,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DefaultAddDataSourceCallbacks; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitJNI; @@ -60,6 +61,7 @@ class AddMultipleImagesTask implements Runnable { private List errorMessages = new ArrayList<>(); private DataSourceProcessorResult result; private List newDataSources = new ArrayList<>(); + private Image currentImage = null; /* * The cancellation requested flag and SleuthKit add image process are @@ -105,6 +107,8 @@ class AddMultipleImagesTask implements Runnable { @Messages({ "AddMultipleImagesTask.cancelled=Cancellation: Add image process reverted", + "# {0} - image path", + "AddMultipleImagesTask.imageError=Error adding image {0} to the database" }) @Override public void run() { @@ -118,15 +122,25 @@ class AddMultipleImagesTask implements Runnable { List corruptedImageFilePaths = new ArrayList<>(); progressMonitor.setIndeterminate(true); for (String imageFilePath : imageFilePaths) { + try { + currentImage = SleuthkitJNI.addImageToDatabase(currentCase.getSleuthkitCase(), new String[]{imageFilePath}, + 0, timeZone, "", "", "", deviceId); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error adding image " + imageFilePath + " to database", ex); + errorMessages.add(Bundle.AddMultipleImagesTask_imageError(imageFilePath)); + result = DataSourceProcessorResult.CRITICAL_ERRORS; + } + synchronized (tskAddImageProcessLock) { + if (!tskAddImageProcessStopped) { addImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(timeZone, false, false, ""); } else { return; } } - run(imageFilePath, corruptedImageFilePaths, errorMessages); - commitOrRevertAddImageProcess(imageFilePath, errorMessages, newDataSources); + run(imageFilePath, currentImage, corruptedImageFilePaths, errorMessages); + finishAddImageProcess(imageFilePath, errorMessages, newDataSources); synchronized (tskAddImageProcessLock) { if (tskAddImageProcessStopped) { errorMessages.add(Bundle.AddMultipleImagesTask_cancelled()); @@ -218,7 +232,8 @@ class AddMultipleImagesTask implements Runnable { /** * Attempts to add an input image to the case. * - * @param imageFilePath The image file path. + * @param imageFilePath Path to the image. + * @param image The image. * @param corruptedImageFilePaths If the image cannot be added because * Sleuth Kit cannot detect a filesystem, * the image file path is added to this list @@ -233,13 +248,13 @@ class AddMultipleImagesTask implements Runnable { "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImagesTask.criticalErrorAdding=Critical error adding {0} for device {1}: {2}", "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImagesTask.criticalErrorReverting=Critical error reverting add image process for {0} for device {1}: {2}", "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImagesTask.nonCriticalErrorAdding=Non-critical error adding {0} for device {1}: {2}",}) - private void run(String imageFilePath, List corruptedImageFilePaths, List errorMessages) { + private void run(String imageFilePath, Image image, List corruptedImageFilePaths, List errorMessages) { /* * Try to add the image to the case database as a data source. */ progressMonitor.setProgressText(Bundle.AddMultipleImagesTask_adding(imageFilePath)); try { - addImageProcess.run(deviceId, new String[]{imageFilePath}); + addImageProcess.run(deviceId, image, 0, new DefaultAddDataSourceCallbacks()); } catch (TskCoreException ex) { if (ex.getMessage().contains(TSK_FS_TYPE_UNKNOWN_ERR_MSG)) { /* @@ -259,9 +274,9 @@ class AddMultipleImagesTask implements Runnable { } /** - * Commits or reverts the results of the TSK add image process. If the - * process was stopped before it completed or there was a critical error the - * results are reverted, otherwise they are committed. + * Finishes TSK add image process. + * The image will always be in the database regardless of whether the user + * canceled or a critical error occurred. * * @param imageFilePath The image file path. * @param errorMessages Error messages, if any, are added to this list for @@ -270,44 +285,26 @@ class AddMultipleImagesTask implements Runnable { * added to this list for eventual return via the * getter method. */ - private void commitOrRevertAddImageProcess(String imageFilePath, List errorMessages, List newDataSources) { - synchronized (tskAddImageProcessLock) { - if (tskAddImageProcessStopped || criticalErrorOccurred) { - try { - addImageProcess.revert(); - } catch (TskCoreException ex) { - errorMessages.add(Bundle.AddMultipleImagesTask_criticalErrorReverting(imageFilePath, deviceId, ex.getLocalizedMessage())); - criticalErrorOccurred = true; - } + private void finishAddImageProcess(String imageFilePath, List errorMessages, List newDataSources) { + synchronized (tskAddImageProcessLock) { + /* + * Add the new image to the list of new data + * sources to be returned via the getter method. + */ + newDataSources.add(currentImage); + + // Do no further processing if the user canceled + if (tskAddImageProcessStopped) { return; } - - /* - * Try to commit the results of the add image process, retrieve the new - * image from the case database, and add it to the list of new data - * sources to be returned via the getter method. - */ - try { - long imageId = addImageProcess.commit(); - Image dataSource = currentCase.getSleuthkitCase().getImageById(imageId); - newDataSources.add(dataSource); - /* - * Verify the size of the new image. Note that it may not be what is - * expected, but at least part of it was added to the case. - */ - String verificationError = dataSource.verifyImageSize(); - if (!verificationError.isEmpty()) { - errorMessages.add(Bundle.AddMultipleImagesTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); - } - } catch (TskCoreException ex) { - /* - * The add image process commit failed or querying the case database - * for the newly added image failed. Either way, this is a critical - * error. - */ - errorMessages.add(Bundle.AddMultipleImagesTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); - criticalErrorOccurred = true; + /* + * Verify the size of the new image. Note that it may not be what is + * expected, but at least part of it was added to the case. + */ + String verificationError = currentImage.verifyImageSize(); + if (!verificationError.isEmpty()) { + errorMessages.add(Bundle.AddMultipleImagesTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); } } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index b12ce63db5..a284e16eb4 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -67,6 +67,8 @@ AddMultipleImagesTask.criticalErrorReverting=Critical error reverting add image # {1} - exceptionMessage AddMultipleImagesTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1} AddMultipleImagesTask.fsTypeUnknownErr=Cannot determine file system type +# {0} - image path +AddMultipleImagesTask.imageError=Error adding image {0} to the database # {0} - imageFilePath # {1} - deviceId # {2} - exceptionMessage diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java index 7a9a446ae6..68487d0fcb 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java @@ -33,7 +33,7 @@ import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; import org.sleuthkit.autopsy.coreutils.NetworkUtils; -import org.sleuthkit.autopsy.ingest.DataSourceIngestJob.Snapshot; +import org.sleuthkit.autopsy.ingest.Snapshot; import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestManager.IngestThreadActivitySnapshot; import org.sleuthkit.autopsy.ingest.IngestProgressSnapshotProvider; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java index c37b834348..288eff8d92 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java @@ -37,7 +37,7 @@ import org.sleuthkit.autopsy.datamodel.NodeProperty; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.Stage; import org.sleuthkit.autopsy.guiutils.DurationCellRenderer; import org.sleuthkit.autopsy.guiutils.StatusIconCellRenderer; -import org.sleuthkit.autopsy.ingest.DataSourceIngestJob; +import org.sleuthkit.autopsy.ingest.Snapshot; /** * A node which represents all AutoIngestJobs of a given AutoIngestJobStatus. @@ -96,7 +96,7 @@ final class AutoIngestJobsNode extends AbstractNode { * they can be changed by events in other threads which */ private final Stage jobStage; - private final List jobSnapshot; + private final List jobSnapshot; private final Integer jobPriority; AutoIngestJobWrapper(AutoIngestJob job) {