From b97bac04830200d259e3f9c96f75b285c27f6c3c Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 6 Nov 2014 02:23:39 -0500 Subject: [PATCH 1/7] Refine and fix multi-stage ingest --- .../DataSourceIngestModuleProgress.java | 10 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 250 +++++++++++------- .../ingest/IngestProgressSnapshotPanel.java | 3 + .../autopsy/ingest/PipelineConfig.xml | 2 +- 4 files changed, 162 insertions(+), 103 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index a972fd86a4..bde810e21d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -71,14 +71,14 @@ public class DataSourceIngestModuleProgress { } /** - * Updates the progress bar with the number of work units performed, if in - * the determinate mode. + * Updates the progress bar with a new task name and the number of work + * units performed, if in the determinate mode. * - * @param message Message to display in sub-title + * @param currentTask The task name. * @param workUnits Number of work units performed so far by the module. */ - public void progress(String message, int workUnits) { - this.job.advanceDataSourceIngestProgressBar(message, workUnits); + public void progress(String currentTask, int workUnits) { + this.job.advanceDataSourceIngestProgressBar(currentTask, workUnits); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index fce16f5bb6..be6566f846 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -47,6 +47,10 @@ final class IngestJob { */ private enum Stages { + /** + * Setting up for processing. + */ + INITIALIZATION, /** * High priority data source ingest modules and file ingest modules. */ @@ -69,22 +73,21 @@ final class IngestJob { private static final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); /** - * These fields define the ingest job and the work it entails. + * These fields define the ingest job and the work it entails. Note that + * there is a collection for multiple copies of the file ingest pipeline, + * one for each file ingest thread. */ private final long id; private final Content dataSource; private final boolean processUnallocatedSpace; private Stages stage; - private DataSourceIngestPipeline dataSourceIngestPipeline; private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; + private DataSourceIngestPipeline currentDataSourceIngestPipeline; private final LinkedBlockingQueue fileIngestPipelines; /** - * These fields are used to update ingest progress UI components for the - * job. The filesInProgress collection contains the names of the files that - * are in the file ingest pipelines and the two file counter fields are used - * to update the file ingest progress bar. + * These fields are used to update ingest progress bars for the job. */ private ProgressHandle dataSourceIngestProgress; private final Object dataSourceIngestProgressLock; @@ -159,7 +162,7 @@ final class IngestJob { static List getJobSnapshots() { List snapShots = new ArrayList<>(); for (IngestJob job : IngestJob.jobsById.values()) { - snapShots.add(job.getIngestJobSnapshot()); + snapShots.add(job.getSnapshot()); } return snapShots; } @@ -185,7 +188,7 @@ final class IngestJob { this.id = id; this.dataSource = dataSource; this.processUnallocatedSpace = processUnallocatedSpace; - this.stage = IngestJob.Stages.FIRST; + this.stage = IngestJob.Stages.INITIALIZATION; this.fileIngestPipelines = new LinkedBlockingQueue<>(); this.filesInProgress = new ArrayList<>(); this.dataSourceIngestProgressLock = new Object(); @@ -222,22 +225,28 @@ final class IngestJob { } /** - * Passes the data source for this job through a data source ingest - * pipeline. + * Passes the data source for this job through the currently active data + * source ingest pipeline. * * @param task A data source ingest task wrapping the data source. */ void process(DataSourceIngestTask task) { try { - if (!this.isCancelled() && !this.dataSourceIngestPipeline.isEmpty()) { + if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + /** + * Run the data source through the pipeline. + */ List errors = new ArrayList<>(); - errors.addAll(this.dataSourceIngestPipeline.process(task)); + errors.addAll(this.currentDataSourceIngestPipeline.process(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } } - // Shut down the data source ingest progress bar right away. + /** + * Shut down the data source ingest progress bar right away. Data + * source-level processing is finished for this stage. + */ synchronized (this.dataSourceIngestProgressLock) { if (null != this.dataSourceIngestProgress) { this.dataSourceIngestProgress.finish(); @@ -245,21 +254,19 @@ final class IngestJob { } } } finally { - // No matter what happens, let the task scheduler know that this - // task is completed and check for job completion. IngestJob.taskScheduler.notifyTaskCompleted(task); - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - this.handleTasksCompleted(); - } + this.checkForCurrentTasksCompleted(); } } /** - * Passes the a file from the data source for this job through the file - * ingest pipeline. + * Passes a file from the data source for this job through the file ingest + * pipeline. * * @param task A file ingest task. - * @throws InterruptedException + * @throws InterruptedException if the thread executing this code is + * interrupted while blocked on taking from or putting to the file ingest + * pipelines collection. */ void process(FileIngestTask task) throws InterruptedException { try { @@ -275,7 +282,9 @@ final class IngestJob { */ AbstractFile file = task.getFile(); - // Update the file ingest progress bar. + /** + * Update the file ingest progress bar. + */ synchronized (this.fileIngestProgressLock) { ++this.processedFiles; if (this.processedFiles <= this.estimatedFilesToProcess) { @@ -286,15 +295,19 @@ final class IngestJob { this.filesInProgress.add(file.getName()); } - // Run the file through the pipeline. + /** + * Run the file through the pipeline. + */ List errors = new ArrayList<>(); errors.addAll(pipeline.process(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } - // Update the file ingest progress bar again in case the - // file was being displayed. + /** + * Update the file ingest progress bar again, in case the + * file was being displayed. + */ if (!this.cancelled) { synchronized (this.fileIngestProgressLock) { this.filesInProgress.remove(file.getName()); @@ -307,17 +320,15 @@ final class IngestJob { } } - // Relinquish the pipeline so it can be reused by another file - // ingest thread. + /** + * Relinquish the pipeline so it can be reused by another file + * ingest thread. + */ this.fileIngestPipelines.put(pipeline); } } finally { - // No matter what happens, let the task scheduler know that this - // task is completed and check for job completion. IngestJob.taskScheduler.notifyTaskCompleted(task); - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - this.handleTasksCompleted(); - } + this.checkForCurrentTasksCompleted(); } } @@ -399,31 +410,31 @@ final class IngestJob { } /** - * Updates the data source ingest progress bar display name. + * Updates the data source ingest progress with a new task name. * - * @param displayName The new display name. + * @param currentTask The task name. */ - void advanceDataSourceIngestProgressBar(String displayName) { + void advanceDataSourceIngestProgressBar(String currentTask) { if (!this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(displayName); + this.dataSourceIngestProgress.progress(currentTask); } } } } /** - * Updates the progress bar with the number of work units performed, if in - * the determinate mode. + * Updates the progress bar with a new task name and the number of work + * units performed, if in the determinate mode. * - * @param message Message to display in sub-title + * @param currentTask The task name. * @param workUnits Number of work units performed. */ - void advanceDataSourceIngestProgressBar(String message, int workUnits) { + void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { if (!this.cancelled) { synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(message, workUnits); + this.dataSourceIngestProgress.progress(currentTask, workUnits); } } } @@ -440,17 +451,19 @@ final class IngestJob { } /** - * Rescind a temporary cancellation of data source ingest in order to stop - * the currently executing data source ingest module. + * Rescind a temporary cancellation of data source ingest used to stop the + * currently executing data source ingest module. */ void currentDataSourceIngestModuleCancellationCompleted() { this.currentDataSourceIngestModuleCancelled = false; - // A new progress bar must be created because the cancel button of the - // previously constructed component is disabled by NetBeans when the - // user selects the "OK" button of the cancellation confirmation dialog - // popped up by NetBeans when the progress bar cancel button was - // pressed. + /** + * A new progress bar must be created because the cancel button of the + * previously constructed component is disabled by NetBeans when the + * user selects the "OK" button of the cancellation confirmation dialog + * popped up by NetBeans when the progress bar cancel button was + * pressed. + */ synchronized (this.dataSourceIngestProgressLock) { this.dataSourceIngestProgress.finish(); this.dataSourceIngestProgress = null; @@ -463,8 +476,10 @@ final class IngestJob { * file ingest pipelines. */ void cancel() { - // Put a cancellation message on data source ingest progress bar, - // if it is still running. + /** + * Put a cancellation message on data source ingest progress bar, if it + * is still running. + */ synchronized (this.dataSourceIngestProgressLock) { if (dataSourceIngestProgress != null) { final String displayName = NbBundle.getMessage(this.getClass(), @@ -477,8 +492,10 @@ final class IngestJob { } } - // Put a cancellation message on the file ingest progress bar, - // if it is still running. + /** + * Put a cancellation message on the file ingest progress bar, if it is + * still running. + */ synchronized (this.fileIngestProgressLock) { if (this.fileIngestProgress != null) { final String displayName = NbBundle.getMessage(this.getClass(), @@ -493,9 +510,11 @@ final class IngestJob { this.cancelled = true; /** - * Tell the task scheduler to cancel all pending tasks. + * Tell the task scheduler to cancel all pending tasks, i.e., tasks not + * not being performed by an ingest thread. */ IngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); + this.checkForCurrentTasksCompleted(); } /** @@ -508,6 +527,24 @@ final class IngestJob { return this.cancelled; } + /** + * Starts up the ingest pipelines and ingest progress bars. + * + * @return A collection of ingest module startup errors, empty on success. + */ + private List start(List ingestModuleTemplates) { + this.createIngestPipelines(ingestModuleTemplates); + List errors = startUpIngestPipelines(); + if (errors.isEmpty()) { + if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { + this.startFirstStage(); + } else if (this.hasSecondStageDataSourceIngestPipeline()) { + this.startSecondStage(); + } + } + return errors; + } + /** * Creates the file and data source ingest pipelines. * @@ -515,7 +552,9 @@ final class IngestJob { * the pipelines. */ private void createIngestPipelines(List ingestModuleTemplates) { - // Make mappings of ingest module factory class names to templates. + /** + * Make mappings of ingest module factory class names to templates. + */ Map dataSourceModuleTemplates = new HashMap<>(); Map fileModuleTemplates = new HashMap<>(); for (IngestModuleTemplate template : ingestModuleTemplates) { @@ -527,16 +566,20 @@ final class IngestJob { } } - // Use the mappings and the ingest pipelines configuration to create - // ordered lists of ingest module templates for each ingest pipeline. + /** + * Use the mappings and the ingest pipelines configuration to create + * ordered lists of ingest module templates for each ingest pipeline. + */ IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); List firstStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); List fileIngestModuleTemplates = this.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); List secondStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); - // Add any module templates that were not specified in the pipeline - // configurations to an appropriate pipeline - either the first stage - // data source ingest pipeline or the file ingest pipeline. + /** + * Add any module templates that were not specified in the pipelines + * configuration to an appropriate pipeline - either the first stage + * data source ingest pipeline or the file ingest pipeline. + */ for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) { firstStageDataSourceModuleTemplates.add(template); } @@ -544,12 +587,15 @@ final class IngestJob { fileIngestModuleTemplates.add(template); } - // Contruct the data source ingest pipelines. + /** + * Construct the data source ingest pipelines. + */ this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - this.dataSourceIngestPipeline = firstStageDataSourceIngestPipeline; - // Construct the file ingest pipelines. + /** + * Construct the file ingest pipelines, one per file ingest thread. + */ try { int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); for (int i = 0; i < numberOfFileIngestThreads; ++i) { @@ -558,8 +604,8 @@ final class IngestJob { } catch (InterruptedException ex) { /** * The current thread was interrupted while blocked on a full queue. - * Blocking should never happen here, but reset the interrupted flag - * rather than just swallowing the exception. + * Blocking should actually never happen here, but reset the + * interrupted flag rather than just swallowing the exception. */ Thread.currentThread().interrupt(); } @@ -567,9 +613,9 @@ final class IngestJob { /** * Use an ordered list of ingest module factory class names to create an - * ordered subset of a collection ingest module templates. The ingest module - * templates are removed from the input collection as they are added to the - * output collection. + * ordered output list of ingest module templates for an ingest pipeline. + * The ingest module templates are removed from the input collection as they + * are added to the output collection. * * @param ingestModuleTemplates A mapping of ingest module factory class * names to ingest module templates. @@ -587,27 +633,6 @@ final class IngestJob { return templates; } - /** - * Starts up the ingest pipelines and ingest progress bars. - * - * @return A collection of ingest module startup errors, empty on success. - */ - private List start(List ingestModuleTemplates) { - this.createIngestPipelines(ingestModuleTemplates); - List errors = startUpIngestPipelines(); - if (errors.isEmpty()) { - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - // There is at least one first stage pipeline. - this.startFirstStage(); - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - // There is no first stage pipeline, but there is a second stage - // ingest pipeline. - this.startSecondStage(); - } - } - return errors; - } - /** * Starts the first stage of the job. */ @@ -624,6 +649,12 @@ final class IngestJob { this.startFileIngestProgressBar(); } + /** + * Make the first stage data source pipeline the current data source + * pipeline. + */ + this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + /** * Schedule the first stage tasks. */ @@ -639,11 +670,9 @@ final class IngestJob { * it is possible, if unlikely, that no file ingest tasks were * actually scheduled since there are files that get filtered out by * the tasks scheduler. In this special case, an ingest thread will - * never get to make the following check for this stage of the job. + * never to check for completion of this stage of the job. */ - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - this.handleTasksCompleted(); - } + this.checkForCurrentTasksCompleted(); } } @@ -653,7 +682,7 @@ final class IngestJob { private void startSecondStage() { this.stage = IngestJob.Stages.SECOND; this.startDataSourceIngestProgressBar(); - this.dataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); } @@ -705,7 +734,7 @@ final class IngestJob { List errors = new ArrayList<>(); // Start up the first stage data source ingest pipeline. - errors.addAll(this.dataSourceIngestPipeline.startUp()); + errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); // Start up the second stage data source ingest pipeline. errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); @@ -795,6 +824,16 @@ final class IngestJob { } } + /** + * Checks to see if the ingest tasks for the current stage are completed and + * invokes a handler if they are. + */ + private void checkForCurrentTasksCompleted() { + if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { + this.handleTasksCompleted(); + } + } + /** * Handles when all ingest tasks for this job are completed by finishing the * current stage and possibly starting the next stage. @@ -902,7 +941,7 @@ final class IngestJob { * * @return An ingest job statistics object. */ - private IngestJobSnapshot getIngestJobSnapshot() { + private IngestJobSnapshot getSnapshot() { return new IngestJobSnapshot(); } @@ -932,19 +971,36 @@ final class IngestJob { this.estimatedFilesToProcess = IngestJob.this.estimatedFilesToProcess; this.snapShotTime = new Date().getTime(); } + + /** + * Get a snapshot of the tasks currently in progress for this job. + */ this.tasksSnapshot = IngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId); } + /** + * Gets the identifier of the ingest job that is the subject of this + * snapshot. + * + * @return The ingest job id. + */ long getJobId() { return this.jobId; } + /** + * Gets the name of the data source associated with the ingest job that + * is the subject of this snapshot. + * + * @return A data source name string. + */ String getDataSource() { return dataSource; } /** - * Gets files per second throughput since job started. + * Gets files per second throughput since the ingest job that is the + * subject of this snapshot started. * * @return Files processed per second (approximate). */ @@ -953,7 +1009,7 @@ final class IngestJob { } /** - * Gets the the ingest job was started. + * Gets the time the ingest job was started. * * @return The start time as number of milliseconds since January 1, * 1970, 00:00:00 GMT. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 61dd097f6f..792a6a4e7f 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -30,6 +30,9 @@ import javax.swing.table.TableColumn; import org.apache.commons.lang3.time.DurationFormatUtils; import org.openide.util.NbBundle; +/** + * A panel that displays ingest task progress snapshots. + */ public class IngestProgressSnapshotPanel extends javax.swing.JPanel { private final JDialog parent; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml b/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml index 69f9f362c3..9a640f633f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml +++ b/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml @@ -16,7 +16,7 @@ Contains only the core ingest modules that ship with Autopsy --> org.sleuthkit.autopsy.thunderbirdparser.EmailParserModuleFactory org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchDetectorModuleFactory org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory - org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory + org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory From 06e2502119c114c3d704e855d850d25450fc656c Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 6 Nov 2014 19:17:43 -0500 Subject: [PATCH 2/7] Fix thread safety issue in multi-stage ingest job --- .../ingest/DataSourceIngestPipeline.java | 1 + .../sleuthkit/autopsy/ingest/IngestJob.java | 262 +++++++++++------- 2 files changed, 156 insertions(+), 107 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index b603166981..702e3b63b4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -71,6 +71,7 @@ final class DataSourceIngestPipeline { "IngestJob.progress.dataSourceIngest.displayName", module.getDisplayName(), dataSource.getName()); this.job.updateDataSourceIngestProgressBarDisplayName(displayName); + this.job.switchDataSourceIngestProgressBarToIndeterminate(); ingestManager.setIngestTaskProgress(task, module.getDisplayName()); module.process(dataSource, new DataSourceIngestModuleProgress(this.job)); } catch (Exception ex) { // Catch-all exception firewall diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index be6566f846..8569bab9b7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -42,26 +42,12 @@ import org.sleuthkit.datamodel.Content; */ final class IngestJob { - /** - * An ingest job may have multiple stages. - */ - private enum Stages { - - /** - * Setting up for processing. - */ - INITIALIZATION, - /** - * High priority data source ingest modules and file ingest modules. - */ - FIRST, - /** - * Lower priority, usually long-running, data source ingest modules. - */ - SECOND - }; - private static final Logger logger = Logger.getLogger(IngestJob.class.getName()); + + /** + * The task scheduler singleton is responsible for creating and scheduling + * the ingest tasks that make up ingest jobs. + */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); /** @@ -73,33 +59,66 @@ final class IngestJob { private static final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); /** - * These fields define the ingest job and the work it entails. Note that - * there is a collection for multiple copies of the file ingest pipeline, - * one for each file ingest thread. + * These fields define the ingest job, including its ingest pipelines. Note + * that there is a collection of multiple copies of the file ingest + * pipeline, one for each file ingest thread. */ private final long id; private final Content dataSource; private final boolean processUnallocatedSpace; - private Stages stage; + private final Object dataSourceIngestPipelineLock; private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; private DataSourceIngestPipeline currentDataSourceIngestPipeline; private final LinkedBlockingQueue fileIngestPipelines; /** - * These fields are used to update ingest progress bars for the job. + * An ingest runs in stages. + */ + private static enum Stages { + + /** + * Setting up for processing. + */ + INITIALIZATION, + /** + * Running high priority data source level ingest modules and file level + * ingest modules. + */ + FIRST, + /** + * Running lower priority, usually long-running, data source level + * ingest modules. + */ + SECOND, + /** + * Cleaning up. + */ + FINALIZATION + }; + private Stages stage; + private final Object stageCompletionCheckLock; + + /** + * These fields are used to provide data source level task progress bars for + * the job. */ - private ProgressHandle dataSourceIngestProgress; private final Object dataSourceIngestProgressLock; + private ProgressHandle dataSourceIngestProgress; + + /** + * These fields are used to provide file level ingest task progress bars for + * the job. + */ + private final Object fileIngestProgressLock; private final List filesInProgress; private long estimatedFilesToProcess; private long processedFiles; private ProgressHandle fileIngestProgress; - private final Object fileIngestProgressLock; /** * These fields support cancellation of either the currently running data - * source ingest module or the entire ingest job. + * source level ingest module or the entire ingest job. */ private volatile boolean currentDataSourceIngestModuleCancelled; private volatile boolean cancelled; @@ -188,11 +207,13 @@ final class IngestJob { this.id = id; this.dataSource = dataSource; this.processUnallocatedSpace = processUnallocatedSpace; - this.stage = IngestJob.Stages.INITIALIZATION; + this.dataSourceIngestPipelineLock = new Object(); this.fileIngestPipelines = new LinkedBlockingQueue<>(); this.filesInProgress = new ArrayList<>(); this.dataSourceIngestProgressLock = new Object(); this.fileIngestProgressLock = new Object(); + this.stage = IngestJob.Stages.INITIALIZATION; + this.stageCompletionCheckLock = new Object(); this.startTime = new Date().getTime(); } @@ -208,15 +229,15 @@ final class IngestJob { /** * Gets the data source to be ingested by this job. * - * @return A reference to a Content object representing the data source. + * @return A Content object representing the data source. */ Content getDataSource() { return this.dataSource; } /** - * Queries whether or not unallocated space should be processed as part of - * this job. + * Gets whether or not unallocated space should be processed as part of this + * job. * * @return True or false. */ @@ -226,20 +247,22 @@ final class IngestJob { /** * Passes the data source for this job through the currently active data - * source ingest pipeline. + * source level ingest pipeline. * * @param task A data source ingest task wrapping the data source. */ void process(DataSourceIngestTask task) { try { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - /** - * Run the data source through the pipeline. - */ - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.process(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); + synchronized (this.dataSourceIngestPipelineLock) { + if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { + /** + * Run the data source through the pipeline. + */ + List errors = new ArrayList<>(); + errors.addAll(this.currentDataSourceIngestPipeline.process(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } } } @@ -254,14 +277,17 @@ final class IngestJob { } } } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForCurrentTasksCompleted(); + this.checkForStageCompleted(); } } /** - * Passes a file from the data source for this job through the file ingest - * pipeline. + * Passes a file from the data source for this job through the file level + * ingest pipeline. * * @param task A file ingest task. * @throws InterruptedException if the thread executing this code is @@ -327,18 +353,26 @@ final class IngestJob { this.fileIngestPipelines.put(pipeline); } } finally { + /** + * No matter what happens, do ingest task bookkeeping. + */ IngestJob.taskScheduler.notifyTaskCompleted(task); - this.checkForCurrentTasksCompleted(); + this.checkForStageCompleted(); } } /** - * Adds more files to an ingest job, i.e., derived or carved files. Not + * Adds more files to an ingest job, i.e., extracted or carved files. Not * currently supported for the second stage of the job. * * @param files A list of files to add. */ void addFiles(List files) { + /** + * Note: This implementation assumes that this is being called by an an + * ingest module running code on an ingest thread that is holding a + * reference to an ingest task, so no task completion check is done. + */ if (IngestJob.Stages.FIRST == this.stage) { for (AbstractFile file : files) { IngestJob.taskScheduler.scheduleFileIngestTask(this, file); @@ -346,10 +380,18 @@ final class IngestJob { } else { IngestJob.logger.log(Level.SEVERE, "Adding files during second stage not supported"); //NON-NLS } + + /** + * The intended clients of this method are ingest modules running code + * on an ingest thread that is holding a reference to an ingest task, in + * which case a task completion check would not be necessary. This is a + * bit of defensive programming. + */ + this.checkForStageCompleted(); } /** - * Updates the display name of the data source ingest progress bar. + * Updates the display name of the data source level ingest progress bar. * * @param displayName The new display name. */ @@ -362,8 +404,9 @@ final class IngestJob { } /** - * Switches the data source progress bar to determinate mode. This should be - * called if the total work units to process the data source is known. + * Switches the data source level ingest progress bar to determinate mode. + * This should be called if the total work units to process the data source + * is known. * * @param workUnits Total number of work units for the processing of the * data source. @@ -379,9 +422,9 @@ final class IngestJob { } /** - * Switches the data source ingest progress bar to indeterminate mode. This - * should be called if the total work units to process the data source is - * unknown. + * Switches the data source level ingest progress bar to indeterminate mode. + * This should be called if the total work units to process the data source + * is unknown. */ void switchDataSourceIngestProgressBarToIndeterminate() { if (!this.cancelled) { @@ -394,8 +437,8 @@ final class IngestJob { } /** - * Updates the data source ingest progress bar with the number of work units - * performed, if in the determinate mode. + * Updates the data source level ingest progress bar with the number of work + * units performed, if in the determinate mode. * * @param workUnits Number of work units performed. */ @@ -410,7 +453,8 @@ final class IngestJob { } /** - * Updates the data source ingest progress with a new task name. + * Updates the data source level ingest progress with a new task name, where + * the task name is the "subtitle" under the display name. * * @param currentTask The task name. */ @@ -425,8 +469,9 @@ final class IngestJob { } /** - * Updates the progress bar with a new task name and the number of work - * units performed, if in the determinate mode. + * Updates the data source level ingest progress bar with a new task name + * and the number of work units performed, if in the determinate mode. The + * task name is the "subtitle" under the display name. * * @param currentTask The task name. * @param workUnits Number of work units performed. @@ -440,9 +485,9 @@ final class IngestJob { } /** - * Determines whether or not a temporary cancellation of data source ingest - * in order to stop the currently executing data source ingest module is in - * effect. + * Queries whether or not a temporary cancellation of data source level + * ingest in order to stop the currently executing data source level ingest + * module is in effect. * * @return True or false. */ @@ -451,8 +496,8 @@ final class IngestJob { } /** - * Rescind a temporary cancellation of data source ingest used to stop the - * currently executing data source ingest module. + * Rescind a temporary cancellation of data source level ingest that was + * used to stop a single data source level ingest module. */ void currentDataSourceIngestModuleCancellationCompleted() { this.currentDataSourceIngestModuleCancelled = false; @@ -472,13 +517,13 @@ final class IngestJob { } /** - * Requests cancellation of ingest, i.e., a shutdown of the data source and - * file ingest pipelines. + * Requests cancellation of ingest, i.e., a shutdown of the data source + * level and file level ingest pipelines. */ void cancel() { /** - * Put a cancellation message on data source ingest progress bar, if it - * is still running. + * Put a cancellation message on data source level ingest progress bar, + * if it is still running. */ synchronized (this.dataSourceIngestProgressLock) { if (dataSourceIngestProgress != null) { @@ -493,8 +538,8 @@ final class IngestJob { } /** - * Put a cancellation message on the file ingest progress bar, if it is - * still running. + * Put a cancellation message on the file level ingest progress bar, if + * it is still running. */ synchronized (this.fileIngestProgressLock) { if (this.fileIngestProgress != null) { @@ -514,12 +559,12 @@ final class IngestJob { * not being performed by an ingest thread. */ IngestJob.taskScheduler.cancelPendingTasksForIngestJob(this); - this.checkForCurrentTasksCompleted(); + this.checkForStageCompleted(); } /** * Queries whether or not cancellation of ingest i.e., a shutdown of the - * data source and file ingest pipelines, has been requested + * data source level and file level ingest pipelines, has been requested. * * @return True or false. */ @@ -571,9 +616,9 @@ final class IngestJob { * ordered lists of ingest module templates for each ingest pipeline. */ IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = this.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + List firstStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + List fileIngestModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + List secondStageDataSourceModuleTemplates = IngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); /** * Add any module templates that were not specified in the pipelines @@ -623,7 +668,7 @@ final class IngestJob { * names representing an ingest pipeline. * @return */ - List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { + private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { List templates = new ArrayList<>(); for (String moduleClassName : pipelineConfig) { if (ingestModuleTemplates.containsKey(moduleClassName)) { @@ -640,7 +685,7 @@ final class IngestJob { this.stage = IngestJob.Stages.FIRST; /** - * Start one or both of the first stage progress bars. + * Start one or both of the first stage ingest progress bars. */ if (this.hasFirstStageDataSourceIngestPipeline()) { this.startDataSourceIngestProgressBar(); @@ -650,10 +695,12 @@ final class IngestJob { } /** - * Make the first stage data source pipeline the current data source - * pipeline. + * Make the first stage data source level ingest pipeline the current + * data source level pipeline. */ - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; + } /** * Schedule the first stage tasks. @@ -672,7 +719,7 @@ final class IngestJob { * the tasks scheduler. In this special case, an ingest thread will * never to check for completion of this stage of the job. */ - this.checkForCurrentTasksCompleted(); + this.checkForStageCompleted(); } } @@ -682,7 +729,9 @@ final class IngestJob { private void startSecondStage() { this.stage = IngestJob.Stages.SECOND; this.startDataSourceIngestProgressBar(); - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + synchronized (this.dataSourceIngestPipelineLock) { + this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + } IngestJob.taskScheduler.scheduleDataSourceIngestTask(this); } @@ -698,7 +747,8 @@ final class IngestJob { } /** - * Checks to see if this job has a first stage data source ingest pipeline. + * Checks to see if this job has a first stage data source level ingest + * pipeline. * * @return True or false. */ @@ -707,7 +757,8 @@ final class IngestJob { } /** - * Checks to see if this job has a second stage data source ingest pipeline. + * Checks to see if this job has a second stage data source level ingest + * pipeline. * * @return True or false. */ @@ -716,7 +767,7 @@ final class IngestJob { } /** - * Checks to see if the job has a file ingest pipeline. + * Checks to see if the job has a file level ingest pipeline. * * @return True or false. */ @@ -725,8 +776,8 @@ final class IngestJob { } /** - * Starts up each of the file and data source ingest modules to collect - * possible errors. + * Starts up each of the file and data source level ingest modules to + * collect possible errors. * * @return A collection of ingest module startup errors, empty on success. */ @@ -767,7 +818,7 @@ final class IngestJob { } /** - * Starts the data source ingest progress bar. + * Starts the data source level ingest progress bar. */ private void startDataSourceIngestProgressBar() { synchronized (this.dataSourceIngestProgressLock) { @@ -800,7 +851,7 @@ final class IngestJob { } /** - * Starts the file ingest progress bar. + * Starts the file level ingest progress bar. */ private void startFileIngestProgressBar() { synchronized (this.fileIngestProgressLock) { @@ -826,26 +877,20 @@ final class IngestJob { /** * Checks to see if the ingest tasks for the current stage are completed and - * invokes a handler if they are. + * does a stage transition if they are. */ - private void checkForCurrentTasksCompleted() { - if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { - this.handleTasksCompleted(); - } - } - - /** - * Handles when all ingest tasks for this job are completed by finishing the - * current stage and possibly starting the next stage. - */ - private void handleTasksCompleted() { - switch (this.stage) { - case FIRST: - this.finishFirstStage(); - break; - case SECOND: - this.finish(); - break; + private void checkForStageCompleted() { + synchronized (this.stageCompletionCheckLock) { + if (IngestJob.taskScheduler.tasksForJobAreCompleted(this)) { + switch (this.stage) { + case FIRST: + this.finishFirstStage(); + break; + case SECOND: + this.finish(); + break; + } + } } } @@ -898,6 +943,8 @@ final class IngestJob { * Shuts down the ingest pipelines and progress bars for this job. */ private void finish() { + this.stage = IngestJob.Stages.FINALIZATION; + // Finish the second stage data source ingest progress bar, if it hasn't // already been finished. synchronized (this.dataSourceIngestProgressLock) { @@ -929,8 +976,8 @@ final class IngestJob { } /** - * Requests a temporary cancellation of data source ingest in order to stop - * the currently executing data source ingest module. + * Requests a temporary cancellation of data source level ingest in order to + * stop the currently executing data source ingest module. */ private void cancelCurrentDataSourceIngestModule() { this.currentDataSourceIngestModuleCancelled = true; @@ -943,6 +990,7 @@ final class IngestJob { */ private IngestJobSnapshot getSnapshot() { return new IngestJobSnapshot(); + } /** From 8a459ab2056d7449fa6c6853b94e368b0119d663 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 6 Nov 2014 21:08:37 -0500 Subject: [PATCH 3/7] make report branding work again --- .../org/sleuthkit/autopsy/report/ReportBranding.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java b/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java index 276004478a..a7d06bc10b 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2013 Basis Technology Corp. + * Copyright 2013-2014 Basis Technology Corp. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -64,7 +64,7 @@ public final class ReportBranding implements ReportBrandingProviderI { //TODO use defaults } } - extractGeneratorLogo(); + extractDefaultGeneratorLogo(); getAgencyLogoPath(); getReportTitle(); } @@ -74,7 +74,10 @@ public final class ReportBranding implements ReportBrandingProviderI { return reportsBrandingDir; } - private void extractGeneratorLogo() { + /** + * extract default logo from JAR file to local file. + */ + private void extractDefaultGeneratorLogo() { try { PlatformUtil.extractResourceToUserConfigDir(getClass(), DEFAULT_GENERATOR_LOGO, true); } catch (IOException ex) { @@ -90,7 +93,7 @@ public final class ReportBranding implements ReportBrandingProviderI { @Override public void setGeneratorLogoPath(String path) { - + generatorLogoPath = path; } @Override From cb01eb40cdcd931fac02afdcfd520941dd927c64 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 6 Nov 2014 21:14:26 -0500 Subject: [PATCH 4/7] Make recent activity module cancellable --- .../sleuthkit/autopsy/coreutils/ExecUtil.java | 5 +- ...ampleExecutableDataSourceIngestModule.java | 27 ++-- .../PhotoRecCarverFileIngestModule.java | 2 +- .../sevenzip/SevenZipIngestModule.java | 2 +- .../KeywordSearchIngestModule.java | 3 +- .../autopsy/recentactivity/Chrome.java | 10 +- .../autopsy/recentactivity/ExtractIE.java | 46 +++---- .../recentactivity/ExtractRegistry.java | 120 ++++++------------ .../autopsy/recentactivity/Firefox.java | 14 +- .../recentactivity/RAImageIngestModule.java | 4 +- .../recentactivity/RecentDocumentsByLnk.java | 2 +- .../SearchEngineURLQueryAnalyzer.java | 4 +- 12 files changed, 102 insertions(+), 137 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/ExecUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/ExecUtil.java index 8005609470..22e44f5f80 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/ExecUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/ExecUtil.java @@ -149,7 +149,10 @@ public final class ExecUtil { logger.log(Level.WARNING, "Error occurred when attempting to kill process: {0}", ex.getMessage()); // NON-NLS } } - + + /** + * EVERYTHING FOLLOWING THIS LINE IS DEPRECATED AND SLATED FOR REMOVAL + */ private static final Logger logger = Logger.getLogger(ExecUtil.class.getName()); private Process proc = null; private ExecUtil.StreamToStringRedirect errorStringRedirect = null; diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java index 98e8029932..ebc3c48eca 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java @@ -52,6 +52,7 @@ import org.sleuthkit.autopsy.externalresults.ExternalResults; import org.sleuthkit.autopsy.externalresults.ExternalResultsImporter; import org.sleuthkit.autopsy.externalresults.ExternalResultsXMLParser; import org.sleuthkit.autopsy.ingest.DataSourceIngestModule; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; @@ -73,15 +74,15 @@ public class SampleExecutableDataSourceIngestModule implements DataSourceIngestM private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); private static final String moduleName = SampleExecutableIngestModuleFactory.getModuleName(); - private final String fileInCaseDatabase = "/WINDOWS/system32/ntmsapi.dll"; // Probably - private long jobId; + private final String fileInCaseDatabase = "/WINDOWS/system32/ntmsapi.dll"; // Probably + private IngestJobContext context; private String outputDirPath; private String derivedFileInCaseDatabase; @Override public void startUp(IngestJobContext context) throws IngestModuleException { - jobId = context.getJobId(); - if (refCounter.incrementAndGet(jobId) == 1) { + this.context = context; + if (refCounter.incrementAndGet(context.getJobId()) == 1) { // Create an output directory for this job. outputDirPath = Case.getCurrentCase().getModulesOutputDirAbsPath() + File.separator + moduleName; //NON-NLS File outputDir = new File(outputDirPath); @@ -93,7 +94,7 @@ public class SampleExecutableDataSourceIngestModule implements DataSourceIngestM @Override public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar) { - if (refCounter.get(jobId) == 1) { + if (refCounter.get(context.getJobId()) == 1) { try { // There will be two tasks: data source analysis and import of // the results of the analysis. @@ -108,14 +109,18 @@ public class SampleExecutableDataSourceIngestModule implements DataSourceIngestM // derived files, and reports generated by the analysis. In this // sample ingest module, the generation of the analysis results is // simulated. - String resultsFilePath = outputDirPath + File.separator + String.format("job_%d_results.xml", jobId); + String resultsFilePath = outputDirPath + File.separator + String.format("job_%d_results.xml", context); boolean haveRealExecutable = false; if (haveRealExecutable) { if (dataSource instanceof Image) { Image image = (Image)dataSource; String dataSourcePath = image.getPaths()[0]; - ExecUtil executor = new ExecUtil(); - executor.execute("some.exe", dataSourcePath, resultsFilePath); + List commandLine = new ArrayList<>(); + commandLine.add("some.exe"); + commandLine.add(dataSourcePath); + commandLine.add(resultsFilePath); + ProcessBuilder processBuilder = new ProcessBuilder(commandLine); + ExecUtil.execute(processBuilder, new DataSourceIngestModuleProcessTerminator(context)); } // not a disk image else { @@ -136,7 +141,7 @@ public class SampleExecutableDataSourceIngestModule implements DataSourceIngestM IngestServices.getInstance().postMessage(IngestMessage.createErrorMessage(moduleName, "External Results Import Error", errorInfo.getMessage())); } progressBar.progress(2); - } catch (InterruptedException | ParserConfigurationException | TransformerException | IOException ex) { + } catch (ParserConfigurationException | TransformerException | IOException ex) { Logger logger = IngestServices.getInstance().getLogger(moduleName); logger.log(Level.SEVERE, "Failed to simulate analysis and results import", ex); //NON-NLS return ProcessResult.ERROR; @@ -155,7 +160,7 @@ public class SampleExecutableDataSourceIngestModule implements DataSourceIngestM List filePaths = new ArrayList<>(); String fileContents = "This is a simulated derived file."; for (int i = 0; i < 2; ++i) { - String fileName = String.format("job_%d_derived_file_%d.txt", jobId, i); + String fileName = String.format("job_%d_derived_file_%d.txt", context.getJobId(), i); filePaths.add(generateFile(fileName, fileContents.getBytes())); if (i == 0) { this.derivedFileInCaseDatabase = this.fileInCaseDatabase + "/" + fileName; @@ -168,7 +173,7 @@ public class SampleExecutableDataSourceIngestModule implements DataSourceIngestM List filePaths = new ArrayList<>(); String fileContents = "This is a simulated report."; for (int i = 0; i < 2; ++i) { - String fileName = String.format("job_%d_report_%d.txt", jobId, i); + String fileName = String.format("job_%d_report_%d.txt", context.getJobId(), i); filePaths.add(generateFile(fileName, fileContents.getBytes())); } return filePaths; diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java index aede55d33c..ffb5447804 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/PhotoRecCarverFileIngestModule.java @@ -216,7 +216,7 @@ final class PhotoRecCarverFileIngestModule implements FileIngestModule { PhotoRecCarverOutputParser parser = new PhotoRecCarverOutputParser(outputDirPath); List theList = parser.parse(newAuditFile, id, file); if (theList != null) { // if there were any results from carving, add the unallocated carving event to the reports list. - context.scheduleFiles(new ArrayList<>(theList)); + context.addFilesToJob(new ArrayList<>(theList)); } } catch (IOException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index 1d02f1c5d8..e4efa94892 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -181,7 +181,7 @@ public final class SevenZipIngestModule implements FileIngestModule { //currently sending a single event for all new files services.fireModuleContentEvent(new ModuleContentEvent(abstractFile)); - context.scheduleFiles(unpackedFiles); + context.addFilesToJob(unpackedFiles); } return ProcessResult.OK; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index a2b74a2946..7fafe1abb5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -247,8 +247,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { return; } - if (context.isJobCancelled()) { - logger.log(Level.INFO, "Ingest job cancelled"); //NON-NLS + if (context.fileIngestIsCancelled()) { stop(); return; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java index e1b7f32f9f..abaaf2a84d 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java @@ -128,7 +128,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } @@ -204,7 +204,7 @@ class Chrome extends Extract { logger.log(Level.INFO, "{0}- Now getting Bookmarks from {1}", new Object[]{moduleName, temps}); //NON-NLS File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } @@ -341,7 +341,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } @@ -416,7 +416,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } @@ -504,7 +504,7 @@ class Chrome extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index 735ad9d92a..451164e0b0 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -52,6 +52,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Content; import org.sleuthkit.autopsy.coreutils.PlatformUtil; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.*; @@ -111,7 +112,7 @@ class ExtractIE extends Extract { continue; } - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { break; } @@ -201,7 +202,7 @@ class ExtractIE extends Extract { dataFound = true; for (AbstractFile cookiesFile : cookiesFiles) { - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { break; } if (cookiesFile.getSize() == 0) { @@ -309,7 +310,7 @@ class ExtractIE extends Extract { //indexFileName = "index" + Long.toString(bbart.getArtifactID()) + ".dat"; temps = RAImageIngestModule.getRATempPath(currentCase, "IE") + File.separator + indexFileName; //NON-NLS File datFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { break; } try { @@ -324,6 +325,9 @@ class ExtractIE extends Extract { String filename = "pasco2Result." + indexFile.getId() + ".txt"; //NON-NLS boolean bPascProcSuccess = executePasco(temps, filename); + if (context.dataSourceIngestIsCancelled()) { + return; + } //At this point pasco2 proccessed the index files. //Now fetch the results, parse them and the delete the files. @@ -354,34 +358,26 @@ class ExtractIE extends Extract { */ private boolean executePasco(String indexFilePath, String outputFileName) { boolean success = true; - - Writer writer = null; - ExecUtil execPasco = new ExecUtil(); try { final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName; - logger.log(Level.INFO, "Writing pasco results to: {0}", outputFileFullPath); //NON-NLS - writer = new FileWriter(outputFileFullPath); - execPasco.execute(writer, JAVA_PATH, - "-cp", PASCO_LIB_PATH, //NON-NLS - "isi.pasco2.Main", "-T", "history", indexFilePath ); //NON-NLS + final String errFileFullPath = moduleTempResultsDir + File.separator + outputFileName + ".err"; + logger.log(Level.INFO, "Writing pasco results to: {0}", outputFileFullPath); //NON-NLS + List commandLine = new ArrayList<>(); + commandLine.add(JAVA_PATH); + commandLine.add("-cp"); //NON-NLS + commandLine.add(PASCO_LIB_PATH); + commandLine.add("isi.pasco2.Main"); //NON-NLS + commandLine.add("-T"); + commandLine.add("history"); + commandLine.add(indexFilePath); + ProcessBuilder processBuilder = new ProcessBuilder(commandLine); + processBuilder.redirectOutput(new File(outputFileFullPath)); + processBuilder.redirectError(new File(errFileFullPath)); + ExecUtil.execute(processBuilder, new DataSourceIngestModuleProcessTerminator(context)); // @@@ Investigate use of history versus cache as type. } catch (IOException ex) { success = false; logger.log(Level.SEVERE, "Unable to execute Pasco to process Internet Explorer web history.", ex); //NON-NLS - } catch (InterruptedException ex) { - success = false; - logger.log(Level.SEVERE, "Pasco has been interrupted, failed to extract some web history from Internet Explorer.", ex); //NON-NLS - } - finally { - if (writer != null) { - try { - writer.flush(); - writer.close(); - } catch (IOException ex) { - logger.log(Level.WARNING, "Error closing writer stream after for Pasco result", ex); //NON-NLS - } - } - execPasco.stop(); } return success; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 1448740ff8..fc62e403eb 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -37,6 +37,7 @@ import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.recentactivity.UsbDeviceIdMapper.USBInfo; import org.sleuthkit.datamodel.*; @@ -57,19 +58,15 @@ import org.xml.sax.SAXException; */ class ExtractRegistry extends Extract { - private Logger logger = Logger.getLogger(this.getClass().getName()); + private static final Logger logger = Logger.getLogger(ExtractRegistry.class.getName()); private String RR_PATH; private String RR_FULL_PATH; private boolean rrFound = false; // true if we found the Autopsy-specific version of regripper - private boolean rrFullFound = false; // true if we found the full version of regripper - final private static String MODULE_VERSION = "1.0"; - + private boolean rrFullFound = false; // true if we found the full version of regripper private Content dataSource; private IngestJobContext context; - final private static UsbDeviceIdMapper usbMapper = new UsbDeviceIdMapper(); - //hide public constructor to prevent from instantiation by ingest module loader ExtractRegistry() { moduleName = NbBundle.getMessage(ExtractIE.class, "ExtractRegistry.moduleName.text"); final File rrRoot = InstalledFileLocator.getDefault().locate("rr", ExtractRegistry.class.getPackage().getName(), false); //NON-NLS @@ -169,7 +166,7 @@ class ExtractRegistry extends Extract { continue; } - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { break; } @@ -182,10 +179,9 @@ class ExtractRegistry extends Extract { logger.log(Level.SEVERE, null, ex); } - logger.log(Level.INFO, moduleName + "- Now getting registry information from " + regFileNameLocal); //NON-NLS - RegOutputFiles regOutputFiles = executeRegRip(regFileNameLocal, outputPathBase); - - if (context.isJobCancelled()) { + logger.log(Level.INFO, "{0}- Now getting registry information from {1}", new Object[]{moduleName, regFileNameLocal}); //NON-NLS + RegOutputFiles regOutputFiles = ripRegistryFile(regFileNameLocal, outputPathBase); + if (context.dataSourceIngestIsCancelled()) { break; } @@ -268,9 +264,9 @@ class ExtractRegistry extends Extract { * @param regFilePath Path to local copy of registry * @param outFilePathBase Path to location to save output file to. Base mtimeItem that will be extended on */ - private RegOutputFiles executeRegRip(String regFilePath, String outFilePathBase) { + private RegOutputFiles ripRegistryFile(String regFilePath, String outFilePathBase) { String autopsyType = ""; // Type argument for rr for autopsy-specific modules - String fullType = ""; // Type argument for rr for full set of modules + String fullType; // Type argument for rr for full set of modules RegOutputFiles regOutputFiles = new RegOutputFiles(); @@ -298,78 +294,44 @@ class ExtractRegistry extends Extract { // run the autopsy-specific set of modules if (!autopsyType.isEmpty() && rrFound) { - // TODO - add error messages - Writer writer = null; - ExecUtil execRR = null; - try { - regOutputFiles.autopsyPlugins = outFilePathBase + "-autopsy.txt"; //NON-NLS - logger.log(Level.INFO, "Writing RegRipper results to: " + regOutputFiles.autopsyPlugins); //NON-NLS - writer = new FileWriter(regOutputFiles.autopsyPlugins); - execRR = new ExecUtil(); - execRR.execute(writer, RR_PATH, - "-r", regFilePath, "-f", autopsyType); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, "Unable to RegRipper and process parse some registry files.", ex); //NON-NLS - this.addErrorMessage( - NbBundle.getMessage(this.getClass(), "ExtractRegistry.execRegRip.errMsg.failedAnalyzeRegFile", - this.getName())); - } catch (InterruptedException ex) { - logger.log(Level.SEVERE, "RegRipper has been interrupted, failed to parse registry.", ex); //NON-NLS - this.addErrorMessage( - NbBundle.getMessage(this.getClass(), "ExtractRegistry.execRegRip.errMsg.failedAnalyzeRegFile2", - this.getName())); - } finally { - if (writer != null) { - try { - writer.close(); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error closing output writer after running RegRipper", ex); //NON-NLS - } - } - if (execRR != null) { - execRR.stop(); - } - } + regOutputFiles.autopsyPlugins = outFilePathBase + "-autopsy.txt"; //NON-NLS + String errFilePath = outFilePathBase + "-autopsy.err.txt"; //NON-NLS + logger.log(Level.INFO, "Writing RegRipper results to: {0}", regOutputFiles.autopsyPlugins); //NON-NLS + executeRegRipper(regFilePath, autopsyType, regOutputFiles.autopsyPlugins, errFilePath); + } + if (context.dataSourceIngestIsCancelled()) { + return regOutputFiles; } // run the full set of rr modules if (!fullType.isEmpty() && rrFullFound) { - Writer writer = null; - ExecUtil execRR = null; - try { - regOutputFiles.fullPlugins = outFilePathBase + "-full.txt"; //NON-NLS - logger.log(Level.INFO, "Writing Full RegRipper results to: " + regOutputFiles.fullPlugins); //NON-NLS - writer = new FileWriter(regOutputFiles.fullPlugins); - execRR = new ExecUtil(); - execRR.execute(writer, RR_FULL_PATH, - "-r", regFilePath, "-f", fullType); //NON-NLS - } catch (IOException ex) { - logger.log(Level.SEVERE, "Unable to run full RegRipper and process parse some registry files.", ex); //NON-NLS - this.addErrorMessage( - NbBundle.getMessage(this.getClass(), "ExtractRegistry.execRegRip.errMsg.failedAnalyzeRegFile3", - this.getName())); - } catch (InterruptedException ex) { - logger.log(Level.SEVERE, "RegRipper full has been interrupted, failed to parse registry.", ex); //NON-NLS - this.addErrorMessage( - NbBundle.getMessage(this.getClass(), "ExtractRegistry.execRegRip.errMsg.failedAnalyzeRegFile4", - this.getName())); - } finally { - if (writer != null) { - try { - writer.close(); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error closing output writer after running RegRipper full", ex); //NON-NLS - } - } - if (execRR != null) { - execRR.stop(); - } - } - } - + regOutputFiles.fullPlugins = outFilePathBase + "-full.txt"; //NON-NLS + String errFilePath = outFilePathBase + "-full.err.txt"; //NON-NLS + logger.log(Level.INFO, "Writing Full RegRipper results to: {0}", regOutputFiles.fullPlugins); //NON-NLS + executeRegRipper(regFilePath, fullType, regOutputFiles.fullPlugins, errFilePath); + } return regOutputFiles; } + private void executeRegRipper(String hiveFilePath, String hiveFileType, String outputFile, String errFile) { + try { + logger.log(Level.INFO, "Writing RegRipper results to: {0}", outputFile); //NON-NLS + List commandLine = new ArrayList<>(); + commandLine.add(RR_PATH); + commandLine.add("-r"); //NON-NLS + commandLine.add(hiveFilePath); + commandLine.add("-f"); //NON-NLS + commandLine.add(hiveFileType); + ProcessBuilder processBuilder = new ProcessBuilder(commandLine); + processBuilder.redirectOutput(new File(outputFile)); + processBuilder.redirectError(new File(errFile)); + ExecUtil.execute(processBuilder, new DataSourceIngestModuleProcessTerminator(context)); + } catch (IOException ex) { + logger.log(Level.SEVERE, "Unable to run RegRipper", ex); //NON-NLS + this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractRegistry.execRegRip.errMsg.failedAnalyzeRegFile", this.getName())); + } + } + // @@@ VERIFY that we are doing the right thing when we parse multiple NTUSER.DAT /** * @@ -558,7 +520,7 @@ class ExtractRegistry extends Extract { } break; default: - logger.log(Level.WARNING, "Unercognized node name: " + dataType); + logger.log(Level.WARNING, "Unrecognized node name: {0}", dataType); break; } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index 61497afbb1..1d5b4d679a 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -116,7 +116,7 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } @@ -197,7 +197,7 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } @@ -277,7 +277,7 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } @@ -385,16 +385,16 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } List> tempList = this.dbConnect(temps, downloadQuery); - logger.log(Level.INFO, moduleName + "- Now getting downloads from " + temps + " with " + tempList.size() + "artifacts identified."); //NON-NLS + logger.log(Level.INFO, "{0}- Now getting downloads from {1} with {2}artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); //NON-NLS for (HashMap result : tempList) { - Collection bbattributes = new ArrayList(); + Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), NbBundle.getMessage(this.getClass(), @@ -494,7 +494,7 @@ class Firefox extends Extract { continue; } File dbFile = new File(temps); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { dbFile.delete(); break; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java index 7cbcbf4e23..1600472f43 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java @@ -93,7 +93,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule { for (int i = 0; i < extracters.size(); i++) { Extract extracter = extracters.get(i); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { logger.log(Level.INFO, "Recent Activity has been canceled, quitting before {0}", extracter.getName()); //NON-NLS break; } @@ -161,7 +161,7 @@ public final class RAImageIngestModule implements DataSourceIngestModule { historyMsg.toString()); services.postMessage(inboxMsg); - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { return ProcessResult.OK; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java index 7496e8a888..cac4c1f9d9 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java @@ -79,7 +79,7 @@ class RecentDocumentsByLnk extends Extract { dataFound = true; for (AbstractFile recentFile : recentFiles) { - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { break; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index f322e828c9..5e5e3ab5f7 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -278,7 +278,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { logger.log(Level.INFO, "Processing {0} blackboard artifacts.", listArtifacts.size()); //NON-NLS for (BlackboardArtifact artifact : listArtifacts) { - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { break; //User cancled the process. } @@ -346,7 +346,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { } catch (TskCoreException e) { logger.log(Level.SEVERE, "Encountered error retrieving artifacts for search engine queries", e); //NON-NLS } finally { - if (context.isJobCancelled()) { + if (context.dataSourceIngestIsCancelled()) { logger.info("Operation terminated by user."); //NON-NLS } IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent( From 94845cbdfbe613d0a1395eeecde9a972bbf2b99b Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 6 Nov 2014 21:49:30 -0500 Subject: [PATCH 5/7] Replace some deprecated method calls --- .../sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java | 2 +- .../thunderbirdparser/ThunderbirdMboxFileIngestModule.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java b/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java index 1e52dec332..7101c71dc4 100644 --- a/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java +++ b/ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java @@ -223,7 +223,7 @@ class ScalpelCarverIngestModule implements FileIngestModule { } // reschedule carved files - context.scheduleFiles(new ArrayList(carvedFiles)); + context.addFilesToJob(new ArrayList(carvedFiles)); return ProcessResult.OK; } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 11ec6905a9..d86fc5589f 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -289,7 +289,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { services.fireModuleContentEvent(new ModuleContentEvent(derived)); } } - context.scheduleFiles(derivedFiles); + context.addFilesToJob(derivedFiles); services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG)); } From 2562cd684ec4850217de0b79bc1ec936e2d75db1 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 6 Nov 2014 22:04:23 -0500 Subject: [PATCH 6/7] Fix typos in recent activity module --- .../examples/SampleExecutableDataSourceIngestModule.java | 2 +- .../src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java | 4 ++-- .../org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java | 2 +- .../src/org/sleuthkit/autopsy/recentactivity/Firefox.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java index ebc3c48eca..338e881398 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java @@ -109,7 +109,7 @@ public class SampleExecutableDataSourceIngestModule implements DataSourceIngestM // derived files, and reports generated by the analysis. In this // sample ingest module, the generation of the analysis results is // simulated. - String resultsFilePath = outputDirPath + File.separator + String.format("job_%d_results.xml", context); + String resultsFilePath = outputDirPath + File.separator + String.format("job_%d_results.xml", context.getJobId()); boolean haveRealExecutable = false; if (haveRealExecutable) { if (dataSource instanceof Image) { diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index 451164e0b0..0af6e1786a 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -367,8 +367,8 @@ class ExtractIE extends Extract { commandLine.add("-cp"); //NON-NLS commandLine.add(PASCO_LIB_PATH); commandLine.add("isi.pasco2.Main"); //NON-NLS - commandLine.add("-T"); - commandLine.add("history"); + commandLine.add("-T"); //NON-NLS + commandLine.add("history"); //NON-NLS commandLine.add(indexFilePath); ProcessBuilder processBuilder = new ProcessBuilder(commandLine); processBuilder.redirectOutput(new File(outputFileFullPath)); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index fc62e403eb..1ad3844348 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -58,7 +58,7 @@ import org.xml.sax.SAXException; */ class ExtractRegistry extends Extract { - private static final Logger logger = Logger.getLogger(ExtractRegistry.class.getName()); + private Logger logger = Logger.getLogger(this.getClass().getName()); private String RR_PATH; private String RR_FULL_PATH; private boolean rrFound = false; // true if we found the Autopsy-specific version of regripper diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index 1d5b4d679a..710ba65d90 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -391,7 +391,7 @@ class Firefox extends Extract { } List> tempList = this.dbConnect(temps, downloadQuery); - logger.log(Level.INFO, "{0}- Now getting downloads from {1} with {2}artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); //NON-NLS + logger.log(Level.INFO, "{0}- Now getting downloads from {1} with {2} artifacts identified.", new Object[]{moduleName, temps, tempList.size()}); //NON-NLS for (HashMap result : tempList) { Collection bbattributes = new ArrayList<>(); From f5fa66a48f199b7f952d496af38cb8aa08c22aed Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 7 Nov 2014 07:26:29 -0500 Subject: [PATCH 7/7] another fix for reporting code and added comments --- .../autopsy/report/ReportBranding.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java b/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java index a7d06bc10b..fe83a6223e 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportBranding.java @@ -48,7 +48,14 @@ public final class ReportBranding implements ReportBrandingProviderI { private String reportsBrandingDir; //dir with extracted reports branding resources private static final String MODULE_NAME = ReportBranding.class.getSimpleName(); private static final Logger logger = Logger.getLogger(ReportBranding.class.getName()); - private static String generatorLogoPath; + + // this is static so that it can be set by another object + // before the report is actually made. Entire class should + // probably become singleton. Is set to null until setPath + // is called to specify something other than default. + private static String generatorLogoPath = null; + + private String defaultGeneratorLogoPath; public ReportBranding() { @@ -83,11 +90,15 @@ public final class ReportBranding implements ReportBrandingProviderI { } catch (IOException ex) { logger.log(Level.SEVERE, "Error extracting report branding resource for generator logo ", ex); //NON-NLS } - generatorLogoPath = PlatformUtil.getUserConfigDirectory() + File.separator + DEFAULT_GENERATOR_LOGO; + defaultGeneratorLogoPath = PlatformUtil.getUserConfigDirectory() + File.separator + DEFAULT_GENERATOR_LOGO; } @Override public String getGeneratorLogoPath() { + // if no one called to change the path, use default + if (generatorLogoPath == null) + generatorLogoPath = defaultGeneratorLogoPath; + return generatorLogoPath; } @@ -100,6 +111,10 @@ public final class ReportBranding implements ReportBrandingProviderI { public String getAgencyLogoPath() { String curPath = null; + /* The agency logo code uses these properties to persist changes + * in the logo (within the same process). + * This is different from the generator logo that uses a static variable. + */ curPath = ModuleSettings.getConfigSetting(MODULE_NAME, AGENCY_LOGO_PATH_PROP); //if has been set, validate it's correct, if not set, return null if (curPath != null && new File(curPath).canRead() == false) { @@ -113,6 +128,8 @@ public final class ReportBranding implements ReportBrandingProviderI { @Override public void setAgencyLogoPath(String path) { + // Use properties to persist the logo to use. + // Should use static variable instead ModuleSettings.setConfigSetting(MODULE_NAME, AGENCY_LOGO_PATH_PROP, path); }