mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-12 07:56:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

7673 ngest infrastructure documentation improvements

Browse Source
This commit is contained in:
Richard Cordovano 2021-10-19 14:54:07 -04:00
parent 40be596425
commit 0c1a44ca2f
14 changed files with 168 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java
View File

@ -26,7 +26,7 @@ import org.sleuthkit.datamodel.DataArtifact;
* A pipeline of data artifact ingest modules used to execute data artifact
* ingest tasks for an ingest job.
*/
final class DataArtifactIngestPipeline extends IngestTaskPipeline<DataArtifactIngestTask> {
final class DataArtifactIngestPipeline extends IngestPipeline<DataArtifactIngestTask> {
/**
* Constructs a pipeline of data artifact ingest modules used to execute
@ -37,13 +37,13 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline<DataArtifactIn
* @param moduleTemplates The ingest module templates that define this
* pipeline. May be an empty list.
*/
DataArtifactIngestPipeline(IngestModulePipelines ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
DataArtifactIngestPipeline(IngestJobExecutor ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
super(ingestJobPipeline, moduleTemplates);
}
@Override
Optional<PipelineModule<DataArtifactIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
Optional<IngestTaskPipeline.PipelineModule<DataArtifactIngestTask>> module = Optional.empty();
Optional<IngestPipeline.PipelineModule<DataArtifactIngestTask>> module = Optional.empty();
if (template.isDataArtifactIngestModuleTemplate()) {
DataArtifactIngestModule ingestModule = template.createDataArtifactIngestModule();
module = Optional.of(new DataArtifactIngestPipelineModule(ingestModule, template.getModuleName()));
@ -52,18 +52,18 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline<DataArtifactIn
}
@Override
void prepareForTask(DataArtifactIngestTask task) throws IngestTaskPipelineException {
void prepareForTask(DataArtifactIngestTask task) throws IngestPipelineException {
}
@Override
void cleanUpAfterTask(DataArtifactIngestTask task) throws IngestTaskPipelineException {
void cleanUpAfterTask(DataArtifactIngestTask task) throws IngestPipelineException {
}
/**
* A decorator that adds ingest infrastructure operations to a data artifact
* ingest module.
*/
static final class DataArtifactIngestPipelineModule extends IngestTaskPipeline.PipelineModule<DataArtifactIngestTask> {
static final class DataArtifactIngestPipelineModule extends IngestPipeline.PipelineModule<DataArtifactIngestTask> {
private final DataArtifactIngestModule module;
@ -80,7 +80,7 @@ final class DataArtifactIngestPipeline extends IngestTaskPipeline<DataArtifactIn
}
@Override
void executeTask(IngestModulePipelines ingestJobPipeline, DataArtifactIngestTask task) throws IngestModuleException {
void process(IngestJobExecutor ingestJobPipeline, DataArtifactIngestTask task) throws IngestModuleException {
DataArtifact artifact = task.getDataArtifact();
module.process(artifact);
}

2
Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java
View File

@ -36,7 +36,7 @@ final class DataArtifactIngestTask extends IngestTask {
* task.
* @param artifact The data artifact to be processed.
*/
DataArtifactIngestTask(IngestModulePipelines ingestJobPipeline, DataArtifact artifact) {
DataArtifactIngestTask(IngestJobExecutor ingestJobPipeline, DataArtifact artifact) {
super(ingestJobPipeline);
this.artifact = artifact;
}

4
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java
View File

@ -23,9 +23,9 @@ package org.sleuthkit.autopsy.ingest;
*/
public class DataSourceIngestModuleProgress {
private final IngestModulePipelines ingestJobPipeline;
private final IngestJobExecutor ingestJobPipeline;
DataSourceIngestModuleProgress(IngestModulePipelines pipeline) {
DataSourceIngestModuleProgress(IngestJobExecutor pipeline) {
this.ingestJobPipeline = pipeline;
}

12
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java
View File

@ -29,7 +29,7 @@ import org.sleuthkit.datamodel.Content;
* A pipeline of data source level ingest modules for executing data source
* level ingest tasks for an ingest job.
*/
final class DataSourceIngestPipeline extends IngestTaskPipeline<DataSourceIngestTask> {
final class DataSourceIngestPipeline extends IngestPipeline<DataSourceIngestTask> {
private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName());
private static final IngestManager ingestManager = IngestManager.getInstance();
@ -42,13 +42,13 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline<DataSourceIngest
* @param moduleTemplates The ingest module templates that define this
* pipeline.
*/
DataSourceIngestPipeline(IngestModulePipelines ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
DataSourceIngestPipeline(IngestJobExecutor ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
super(ingestJobPipeline, moduleTemplates);
}
@Override
Optional<IngestTaskPipeline.PipelineModule<DataSourceIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
Optional<IngestTaskPipeline.PipelineModule<DataSourceIngestTask>> module = Optional.empty();
Optional<IngestPipeline.PipelineModule<DataSourceIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
Optional<IngestPipeline.PipelineModule<DataSourceIngestTask>> module = Optional.empty();
if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModule ingestModule = template.createDataSourceIngestModule();
module = Optional.of(new DataSourcePipelineModule(ingestModule, template.getModuleName()));
@ -69,7 +69,7 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline<DataSourceIngest
* A wrapper that adds ingest infrastructure operations to a data source
* level ingest module.
*/
static final class DataSourcePipelineModule extends IngestTaskPipeline.PipelineModule<DataSourceIngestTask> {
static final class DataSourcePipelineModule extends IngestPipeline.PipelineModule<DataSourceIngestTask> {
private final DataSourceIngestModule module;
@ -83,7 +83,7 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline<DataSourceIngest
}
@Override
void executeTask(IngestModulePipelines ingestJobPipeline, DataSourceIngestTask task) throws IngestModuleException {
void process(IngestJobExecutor ingestJobPipeline, DataSourceIngestTask task) throws IngestModuleException {
Content dataSource = task.getDataSource();
String progressBarDisplayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", getDisplayName(), dataSource.getName());
ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(progressBarDisplayName);

2
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java
View File

@ -31,7 +31,7 @@ final class DataSourceIngestTask extends IngestTask {
* @param ingestJobPipeline The ingest job pipeline to use to execute the
* task.
*/
DataSourceIngestTask(IngestModulePipelines ingestJobPipeline) {
DataSourceIngestTask(IngestJobExecutor ingestJobPipeline) {
super(ingestJobPipeline);
}

32
Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java
View File

@ -39,13 +39,13 @@ import org.sleuthkit.datamodel.TskCoreException;
@NbBundle.Messages({
"FileIngestPipeline_SaveResults_Activity=Saving Results"
})
final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
final class FileIngestPipeline extends IngestPipeline<FileIngestTask> {
private static final int FILE_BATCH_SIZE = 500;
private static final String SAVE_RESULTS_ACTIVITY = Bundle.FileIngestPipeline_SaveResults_Activity();
private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName());
private static final IngestManager ingestManager = IngestManager.getInstance();
private final IngestModulePipelines ingestJobPipeline;
private final IngestJobExecutor ingestJobPipeline;
private final List<AbstractFile> fileBatch;
/**
@ -56,15 +56,15 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
* @param moduleTemplates The ingest module templates that define this
* pipeline.
*/
FileIngestPipeline(IngestModulePipelines ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
FileIngestPipeline(IngestJobExecutor ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
super(ingestJobPipeline, moduleTemplates);
this.ingestJobPipeline = ingestJobPipeline;
fileBatch = new ArrayList<>();
}
@Override
Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> module = Optional.empty();
Optional<IngestPipeline.PipelineModule<FileIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
Optional<IngestPipeline.PipelineModule<FileIngestTask>> module = Optional.empty();
if (template.isFileIngestModuleTemplate()) {
FileIngestModule ingestModule = template.createFileIngestModule();
module = Optional.of(new FileIngestPipelineModule(ingestModule, template.getModuleName()));
@ -73,18 +73,18 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
}
@Override
void prepareForTask(FileIngestTask task) throws IngestTaskPipelineException {
void prepareForTask(FileIngestTask task) throws IngestPipelineException {
}
@Override
void cleanUpAfterTask(FileIngestTask task) throws IngestTaskPipelineException {
void cleanUpAfterTask(FileIngestTask task) throws IngestPipelineException {
try {
ingestManager.setIngestTaskProgress(task, SAVE_RESULTS_ACTIVITY);
AbstractFile file = task.getFile();
file.close();
cacheFileForBatchUpdate(file);
} catch (TskCoreException ex) {
throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
throw new IngestPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
} finally {
ingestManager.setIngestTaskProgressCompleted(task);
}
@ -96,7 +96,7 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
Date start = new Date();
try {
updateBatchedFiles();
} catch (IngestTaskPipelineException ex) {
} catch (IngestPipelineException ex) {
errors.add(new IngestModuleError(SAVE_RESULTS_ACTIVITY, ex));
}
Date finish = new Date();
@ -113,9 +113,9 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
*
* @param file The file.
*
* @throws IngestTaskPipelineException if the case database update fails.
* @throws IngestPipelineException if the case database update fails.
*/
private void cacheFileForBatchUpdate(AbstractFile file) throws IngestTaskPipelineException {
private void cacheFileForBatchUpdate(AbstractFile file) throws IngestPipelineException {
/*
* Only one file ingest thread at a time will try to access the file
* cache. The synchronization here is to ensure visibility of the files
@ -134,9 +134,9 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
* Updates the case database with new properties added to the files in the
* cache by the ingest modules that processed them.
*
* @throws IngestTaskPipelineException if the case database update fails.
* @throws IngestPipelineException if the case database update fails.
*/
private void updateBatchedFiles() throws IngestTaskPipelineException {
private void updateBatchedFiles() throws IngestPipelineException {
/*
* Only one file ingest thread at a time will try to access the file
* cache. The synchronization here is to ensure visibility of the files
@ -166,7 +166,7 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
logger.log(Level.SEVERE, "Error rolling back transaction after failure to save updated properties for cached files from tasks", ex1);
}
}
throw new IngestTaskPipelineException("Failed to save updated properties for cached files from tasks", ex); //NON-NLS
throw new IngestPipelineException("Failed to save updated properties for cached files from tasks", ex); //NON-NLS
} finally {
fileBatch.clear();
}
@ -177,7 +177,7 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
* A wrapper that adds ingest infrastructure operations to a file ingest
* module.
*/
static final class FileIngestPipelineModule extends IngestTaskPipeline.PipelineModule<FileIngestTask> {
static final class FileIngestPipelineModule extends IngestPipeline.PipelineModule<FileIngestTask> {
private final FileIngestModule module;
@ -195,7 +195,7 @@ final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
}
@Override
void executeTask(IngestModulePipelines ingestJobPipeline, FileIngestTask task) throws IngestModuleException {
void process(IngestJobExecutor ingestJobPipeline, FileIngestTask task) throws IngestModuleException {
AbstractFile file = null;
try {
file = task.getFile();

8
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java
View File

@ -40,7 +40,7 @@ final class FileIngestTask extends IngestTask {
* task.
* @param file The file to be processed.
*/
FileIngestTask(IngestModulePipelines ingestJobPipeline, AbstractFile file) {
FileIngestTask(IngestJobExecutor ingestJobPipeline, AbstractFile file) {
super(ingestJobPipeline);
this.file = file;
fileId = file.getId();
@ -56,7 +56,7 @@ final class FileIngestTask extends IngestTask {
* task.
* @param fileId The object ID of the file to be processed.
*/
FileIngestTask(IngestModulePipelines ingestJobPipeline, long fileId) {
FileIngestTask(IngestJobExecutor ingestJobPipeline, long fileId) {
super(ingestJobPipeline);
this.fileId = fileId;
}
@ -100,8 +100,8 @@ final class FileIngestTask extends IngestTask {
return false;
}
FileIngestTask other = (FileIngestTask) obj;
IngestModulePipelines thisPipeline = getIngestJobPipeline();
IngestModulePipelines otherPipeline = other.getIngestJobPipeline();
IngestJobExecutor thisPipeline = getIngestJobPipeline();
IngestJobExecutor otherPipeline = other.getIngestJobPipeline();
if (thisPipeline != otherPipeline && (thisPipeline == null || !thisPipeline.equals(otherPipeline))) {
return false;
}

8
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -73,7 +73,7 @@ public final class IngestJob {
private final List<AbstractFile> files = new ArrayList<>();
private final Mode ingestMode;
private final IngestJobSettings settings;
private volatile IngestModulePipelines ingestModulePipelines;
private volatile IngestJobExecutor ingestModulePipelines;
private volatile CancellationReason cancellationReason;
/**
@ -181,7 +181,7 @@ public final class IngestJob {
return Collections.emptyList();
}
ingestModulePipelines = new IngestModulePipelines(this, dataSource, files, settings);
ingestModulePipelines = new IngestJobExecutor(this, dataSource, files, settings);
List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(ingestModulePipelines.startUp());
if (errors.isEmpty()) {
@ -507,7 +507,7 @@ public final class IngestJob {
*/
public static class DataSourceIngestModuleHandle {
private final IngestModulePipelines ingestJobPipeline;
private final IngestJobExecutor ingestJobPipeline;
private final DataSourceIngestPipeline.DataSourcePipelineModule module;
private final boolean cancelled;
@ -520,7 +520,7 @@ public final class IngestJob {
* source level ingest module.
* @param module The data source level ingest module.
*/
private DataSourceIngestModuleHandle(IngestModulePipelines ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) {
private DataSourceIngestModuleHandle(IngestJobExecutor ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) {
this.ingestJobPipeline = ingestJobPipeline;
this.module = module;
this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled();

4
Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java
View File

@ -29,7 +29,7 @@ import org.sleuthkit.datamodel.DataArtifact;
*/
public final class IngestJobContext {
private final IngestModulePipelines ingestJobPipeline;
private final IngestJobExecutor ingestJobPipeline;
/**
* Constructs an ingest job context object that provides an ingest module
@ -37,7 +37,7 @@ public final class IngestJobContext {
*
* @param ingestJobPipeline The ingest pipeline for the job.
*/
IngestJobContext(IngestModulePipelines ingestJobPipeline) {
IngestJobContext(IngestJobExecutor ingestJobPipeline) {
this.ingestJobPipeline = ingestJobPipeline;
}

40
Core/src/org/sleuthkit/autopsy/ingest/IngestModulePipelines.java → Core/src/org/sleuthkit/autopsy/ingest/IngestJobExecutor.java
View File

@ -59,10 +59,10 @@ import org.sleuthkit.datamodel.DataSource;
* Manages the construction, start up, execution, and shut down of the ingest
* module pipelines for an ingest job.
*/
final class IngestModulePipelines {
final class IngestJobExecutor {
private static final String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy";
private static final Logger logger = Logger.getLogger(IngestModulePipelines.class.getName());
private static final Logger logger = Logger.getLogger(IngestJobExecutor.class.getName());
/*
* A regular expression for identifying the proxy classes Jython generates
@ -159,7 +159,7 @@ final class IngestModulePipelines {
* So the stage transition lock is used not to guard the stage field, but to
* coordinate stage transitions.
*/
private volatile IngestJobStages stage = IngestModulePipelines.IngestJobStages.PIPELINES_START_UP;
private volatile IngestJobStages stage = IngestJobExecutor.IngestJobStages.PIPELINES_START_UP;
private final Object stageTransitionLock = new Object();
/*
@ -238,7 +238,7 @@ final class IngestModulePipelines {
* @throws InterruptedException Exception thrown if the thread in which the
* pipeline is being created is interrupted.
*/
IngestModulePipelines(IngestJob ingestJob, Content dataSource, List<AbstractFile> files, IngestJobSettings settings) throws InterruptedException {
IngestJobExecutor(IngestJob ingestJob, Content dataSource, List<AbstractFile> files, IngestJobSettings settings) throws InterruptedException {
if (!(dataSource instanceof DataSource)) {
throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS
}
@ -594,7 +594,7 @@ final class IngestModulePipelines {
*
* @return A list of ingest module startup errors, empty on success.
*/
private List<IngestModuleError> startUpIngestModulePipeline(IngestTaskPipeline<?> pipeline) {
private List<IngestModuleError> startUpIngestModulePipeline(IngestPipeline<?> pipeline) {
List<IngestModuleError> startUpErrors = pipeline.startUp();
if (!startUpErrors.isEmpty()) {
List<IngestModuleError> shutDownErrors = pipeline.shutDown();
@ -776,7 +776,7 @@ final class IngestModulePipelines {
void startStreamingModeDataSrcAnalysis() {
synchronized (stageTransitionLock) {
logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS
stage = IngestModulePipelines.IngestJobStages.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS;
stage = IngestJobExecutor.IngestJobStages.FILE_AND_HIGH_PRIORITY_DATA_SRC_LEVEL_ANALYSIS;
currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline;
if (hasFileIngestModules()) {
@ -809,7 +809,7 @@ final class IngestModulePipelines {
currentDataSourceIngestPipeline = highPriorityDataSourceIngestPipeline;
if (hasHighPriorityDataSourceIngestModules()) {
IngestModulePipelines.taskScheduler.scheduleDataSourceIngestTask(this);
IngestJobExecutor.taskScheduler.scheduleDataSourceIngestTask(this);
} else {
/*
* If no data source level ingest task is scheduled at this time
@ -831,7 +831,7 @@ final class IngestModulePipelines {
synchronized (stageTransitionLock) {
if (hasLowPriorityDataSourceIngestModules()) {
logInfoMessage(String.format("Starting low priority data source analysis for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), ingestJob.getId())); //NON-NLS
stage = IngestModulePipelines.IngestJobStages.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS;
stage = IngestJobExecutor.IngestJobStages.LOW_PRIORITY_DATA_SRC_LEVEL_ANALYSIS;
if (usingNetBeansGUI) {
startDataSourceIngestProgressBar();
@ -857,7 +857,7 @@ final class IngestModulePipelines {
artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
IngestModulePipelines.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
return true;
}
});
@ -891,12 +891,12 @@ final class IngestModulePipelines {
* ingest job.
*/
DataSourceIngestCancellationPanel panel = new DataSourceIngestCancellationPanel();
String dialogTitle = NbBundle.getMessage(IngestModulePipelines.this.getClass(), "IngestJob.cancellationDialog.title");
String dialogTitle = NbBundle.getMessage(IngestJobExecutor.this.getClass(), "IngestJob.cancellationDialog.title");
JOptionPane.showConfirmDialog(WindowManager.getDefault().getMainWindow(), panel, dialogTitle, JOptionPane.OK_OPTION, JOptionPane.PLAIN_MESSAGE);
if (panel.cancelAllDataSourceIngestModules()) {
IngestModulePipelines.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
} else {
IngestModulePipelines.this.cancelCurrentDataSourceIngestModule();
IngestJobExecutor.this.cancelCurrentDataSourceIngestModule();
}
return true;
}
@ -921,7 +921,7 @@ final class IngestModulePipelines {
fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
IngestModulePipelines.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
IngestJobExecutor.this.cancel(IngestJob.CancellationReason.USER_CANCELLED);
return true;
}
});
@ -998,7 +998,7 @@ final class IngestModulePipelines {
private void shutDown() {
synchronized (stageTransitionLock) {
logInfoMessage("Finished all tasks"); //NON-NLS
stage = IngestModulePipelines.IngestJobStages.PIPELINES_SHUT_DOWN;
stage = IngestJobExecutor.IngestJobStages.PIPELINES_SHUT_DOWN;
shutDownIngestModulePipeline(currentDataSourceIngestPipeline);
shutDownIngestModulePipeline(artifactIngestPipeline);
@ -1056,7 +1056,7 @@ final class IngestModulePipelines {
*
* @param pipeline The pipeline.
*/
private <T extends IngestTask> void shutDownIngestModulePipeline(IngestTaskPipeline<T> pipeline) {
private <T extends IngestTask> void shutDownIngestModulePipeline(IngestPipeline<T> pipeline) {
if (pipeline.isRunning()) {
List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(pipeline.shutDown());
@ -1076,7 +1076,7 @@ final class IngestModulePipelines {
try {
if (!isCancelled()) {
List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(currentDataSourceIngestPipeline.executeTask(task));
errors.addAll(currentDataSourceIngestPipeline.performTask(task));
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
@ -1130,7 +1130,7 @@ final class IngestModulePipelines {
* Run the file through the modules in the pipeline.
*/
List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(pipeline.executeTask(task));
errors.addAll(pipeline.performTask(task));
if (!errors.isEmpty()) {
logIngestModuleErrors(errors, file);
}
@ -1171,7 +1171,7 @@ final class IngestModulePipelines {
try {
if (!isCancelled() && !artifactIngestPipeline.isEmpty()) {
List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(artifactIngestPipeline.executeTask(task));
errors.addAll(artifactIngestPipeline.performTask(task));
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
@ -1191,7 +1191,7 @@ final class IngestModulePipelines {
void addStreamedFiles(List<Long> fileObjIds) {
if (hasFileIngestModules()) {
if (stage.equals(IngestJobStages.STREAMED_FILE_ANALYSIS_ONLY)) {
IngestModulePipelines.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds);
IngestJobExecutor.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds);
} else {
logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported");
}
@ -1412,7 +1412,7 @@ final class IngestModulePipelines {
void cancel(IngestJob.CancellationReason reason) {
jobCancelled = true;
cancellationReason = reason;
IngestModulePipelines.taskScheduler.cancelPendingFileTasksForIngestJob(this);
IngestJobExecutor.taskScheduler.cancelPendingFileTasksForIngestJob(this);
if (usingNetBeansGUI) {
synchronized (dataSourceIngestProgressLock) {

187
Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java → Core/src/org/sleuthkit/autopsy/ingest/IngestPipeline.java
View File

@ -33,21 +33,24 @@ import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
/**
* An abstract superclass for pipelines of ingest modules that execute ingest
* tasks for an ingest job. Subclasses need to extend this class and to
* implement a specialization of the inner PipelineModule abstract superclass.
* An abstract superclass for pipelines of ingest modules that perform the
* ingest tasks that make up an ingest job. A pipeline performs a task by
* passing it sequentially to the process() method of each module in the
* pipeline.
*
* NOTE ON MULTI-THREADING POLICY: This class is primarily designed for use
* by one thread at a time. There are a few status fields that are volatile to
* ensure visibility to threads making ingest progress snapshots, but methods
* such as startUp(), executeTask() and shutDown() are not synchronized.
*
* @param <T> The ingest task type.
* @param <T> The type of ingest tasks the pipeline performs.
*/
abstract class IngestTaskPipeline<T extends IngestTask> {
abstract class IngestPipeline<T extends IngestTask> {
private static final Logger logger = Logger.getLogger(IngestTaskPipeline.class.getName());
private final IngestModulePipelines ingestJobPipeline;
/*
* NOTE ON MULTI-THREADING POLICY: This class is primarily designed for use
* by one thread at a time. There are a few status fields that are volatile
* to ensure visibility to threads making ingest progress snapshots, but
* methods such as startUp(), performTask() and shutDown() are not
* synchronized.
*/
private static final Logger logger = Logger.getLogger(IngestPipeline.class.getName());
private final IngestJobExecutor ingestJobExecutor;
private final List<IngestModuleTemplate> moduleTemplates;
private final List<PipelineModule<T>> modules;
private volatile Date startTime;
@ -56,38 +59,34 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
/**
* Constructs the superclass part of a pipeline of ingest modules that
* executes ingest tasks for an ingest job.
* performs ingest tasks for an ingest job.
*
* @param ingestPipeline The parent ingest job pipeline for this ingest
* task pipeline.
* @param moduleTemplates The ingest module templates that define this
* ingest task pipeline. May be an empty list.
* @param ingestJobExecutor The ingest job executor that owns this pipeline.
* @param moduleTemplates The ingest module templates to be used to
* construct the ingest modules for this pipeline.
* May be an empty list if this type of pipeline is
* not needed for the ingest job.
*/
IngestTaskPipeline(IngestModulePipelines ingestPipeline, List<IngestModuleTemplate> moduleTemplates) {
this.ingestJobPipeline = ingestPipeline;
/*
* The creation of ingest modules from the ingest module templates has
* been deliberately deferred to the startUp() method so that any and
* all errors in module construction or start up can be reported to the
* client code.
*/
IngestPipeline(IngestJobExecutor ingestJobExecutor, List<IngestModuleTemplate> moduleTemplates) {
this.ingestJobExecutor = ingestJobExecutor;
this.moduleTemplates = moduleTemplates;
modules = new ArrayList<>();
}
/**
* Indicates whether or not there are any ingest modules in this ingest task
* Indicates whether or not there are any ingest modules in this ingest
* pipeline.
*
* @return True or false.
* @return True or false; always true before startUp() is called.
*/
boolean isEmpty() {
return modules.isEmpty();
}
/**
* Queries whether or not this ingest task pipeline is running, i.e., the
* startUp() method has been called and the shutDown() has not been called.
* Queries whether or not this ingest pipeline is running, i.e., the
* startUp() method has been called and the shutDown() method has not been
* called yet.
*
* @return True or false.
*/
@ -96,8 +95,8 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
}
/**
* Starts up this ingest task pipeline by calling the startUp() methods of
* the ingest modules in the pipeline.
* Starts up this ingest pipeline by calling the startUp() methods of the
* ingest modules in the pipeline.
*
* @return A list of ingest module start up errors, possibly empty.
*/
@ -110,21 +109,19 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
* any and all errors in module construction or start up can be
* reported to the client code.
*/
createIngestModules(moduleTemplates);
createIngestModules();
errors.addAll(startUpIngestModules());
} else {
errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestTaskPipelineException("Pipeline already started"))); //NON-NLS
errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestPipelineException("Pipeline already started"))); //NON-NLS
}
return errors;
}
/**
* Creates the ingest modules for this ingest task pipeline from the given
* ingest module templates.
*
* @param moduleTemplates The ingest module templates.
* Creates the ingest modules for this ingest pipeline using its ingest
* module templates.
*/
private void createIngestModules(List<IngestModuleTemplate> moduleTemplates) {
private void createIngestModules() {
if (modules.isEmpty()) {
for (IngestModuleTemplate template : moduleTemplates) {
Optional<PipelineModule<T>> module = acceptModuleTemplate(template);
@ -137,8 +134,8 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
/**
* Determines if one of the types of ingest modules that can be created from
* a given ingest module template should be added to this ingest task
* pipeline. If so, the ingest module is created and returned.
* a given ingest module template should be added to this ingest pipeline.
* If so, the ingest module is created and returned.
*
* @param template The ingest module template to be used or ignored, as
* appropriate to the pipeline type.
@ -149,7 +146,7 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
abstract Optional<PipelineModule<T>> acceptModuleTemplate(IngestModuleTemplate template);
/**
* Starts up the ingest modules in this ingest task pipeline.
* Starts up the ingest modules in this ingest pipeline.
*
* @return A list of ingest module start up errors, possibly empty.
*/
@ -159,7 +156,7 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
running = true;
for (PipelineModule<T> module : modules) {
try {
module.startUp(new IngestJobContext(ingestJobPipeline));
module.startUp(new IngestJobContext(ingestJobExecutor));
} catch (Throwable ex) {
/*
* A catch-all exception firewall. Start up errors for all of
@ -174,10 +171,10 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
}
/**
* Returns the start up time of this ingest task pipeline.
* Returns the start up time of this ingest pipeline.
*
* @return The file processing start time, may be null if this pipeline has
* not been started yet.
* @return The start up time, may be null if this pipeline has not been
* started yet.
*/
Date getStartTime() {
Date reportedStartTime = null;
@ -188,65 +185,66 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
}
/**
* Executes an ingest task by calling the process() methods of the ingest
* modules in this ingest task pipeline.
* Performs an ingest task by sequentially calling the process() methods of
* the ingest modules in this ingest pipeline.
*
* @param task The task.
*
* @return A list of ingest module task processing errors, possibly empty.
* @return A list of ingest module processing errors, possibly empty.
*/
List<IngestModuleError> executeTask(T task) {
List<IngestModuleError> performTask(T task) {
List<IngestModuleError> errors = new ArrayList<>();
if (running) {
if (!ingestJobPipeline.isCancelled()) {
if (!ingestJobExecutor.isCancelled()) {
pauseIfScheduled();
if (ingestJobPipeline.isCancelled()) {
if (ingestJobExecutor.isCancelled()) {
return errors;
}
try {
prepareForTask(task);
} catch (IngestTaskPipelineException ex) {
} catch (IngestPipelineException ex) {
errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS
return errors;
}
for (PipelineModule<T> module : modules) {
pauseIfScheduled();
if (ingestJobPipeline.isCancelled()) {
if (ingestJobExecutor.isCancelled()) {
break;
}
try {
currentModule = module;
currentModule.setProcessingStartTime();
module.executeTask(ingestJobPipeline, task);
} catch (Throwable ex) {
module.process(ingestJobExecutor, task);
} catch (Throwable ex) { // Catch-all exception firewall
/*
* A catch-all exception firewall. Note that a runtime
* exception from a single module does not stop
* Note that an exception from a module does not stop
* processing of the task by the other modules in the
* pipeline.
*/
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (ingestJobPipeline.isCancelled()) {
if (ingestJobExecutor.isCancelled()) {
break;
}
}
}
try {
cleanUpAfterTask(task);
} catch (IngestTaskPipelineException ex) {
} catch (IngestPipelineException ex) {
errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS
}
} else {
errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestTaskPipelineException("Pipeline not started or shut down"))); //NON-NLS
errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestPipelineException("Pipeline not started or shut down"))); //NON-NLS
}
currentModule = null;
return errors;
}
/**
* Pauses task execution if ingest has been configured to be paused weekly
* at a specified time for a specified duration.
* Pauses this pipeline if ingest has been configured to be paused weekly at
* a specified time, for a specified duration. A pipeline can only be paused
* between calls to module process() methods, i.e., the individual modules
* themselves cannot be paused in the middle of processing a task.
*/
private void pauseIfScheduled() {
if (ScheduledIngestPauseSettings.getPauseEnabled() == true) {
@ -278,7 +276,7 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
*/
LocalDateTime timeNow = LocalDateTime.now();
if ((timeNow.equals(pauseStart) || timeNow.isAfter(pauseStart)) && timeNow.isBefore(pauseEnd)) {
ingestJobPipeline.registerPausedIngestThread(Thread.currentThread());
ingestJobExecutor.registerPausedIngestThread(Thread.currentThread());
try {
long timeRemainingMillis = ChronoUnit.MILLIS.between(timeNow, pauseEnd);
logger.log(Level.INFO, String.format("%s pausing at %s for ~%d minutes", Thread.currentThread().getName(), LocalDateTime.now(), TimeUnit.MILLISECONDS.toMinutes(timeRemainingMillis)));
@ -287,27 +285,27 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
} catch (InterruptedException notLogged) {
logger.log(Level.INFO, String.format("%s resuming at %s due to sleep interrupt (ingest job canceled)", Thread.currentThread().getName(), LocalDateTime.now()));
} finally {
ingestJobPipeline.unregisterPausedIngestThread(Thread.currentThread());
ingestJobExecutor.unregisterPausedIngestThread(Thread.currentThread());
}
}
}
}
/**
* Does any task type specific preparation required before executing an
* Does any task-type-specific preparation required before performing an
* ingest task.
*
* @param task The task.
*
* @throws IngestTaskPipelineException Thrown if there is an error preparing
* to execute the task.
* @throws IngestPipelineException Thrown if there is an error preparing to
* perform the task.
*/
abstract void prepareForTask(T task) throws IngestTaskPipelineException;
abstract void prepareForTask(T task) throws IngestPipelineException;
/**
* Gets the currently running ingest module.
*
* @return The module, possibly null if no module is currently running.
* @return The module, possibly null, if no module is currently running.
*/
PipelineModule<T> getCurrentlyRunningModule() {
return currentModule;
@ -345,22 +343,22 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
}
/**
* Does any task type specific clean up required after executing an ingest
* Does any task-type-specific clean up required after performing an ingest
* task.
*
* @param task The task.
*
* @throws IngestTaskPipelineException Thrown if there is an error cleaning
* up after performing the task.
* @throws IngestPipelineException Thrown if there is an error cleaning up
* after performing the task.
*/
abstract void cleanUpAfterTask(T task) throws IngestTaskPipelineException;
abstract void cleanUpAfterTask(T task) throws IngestPipelineException;
/**
* An abstract superclass for a decorator that adds ingest infrastructure
* operations to an ingest module.
*
* IMPORTANT: Subclasses of IngestTaskPipeline need to implement a
* specialization this class
* An abstract superclass for an ingest module decorator that adds ingest
* infrastructure operations to the process() method of an ingest module.
* Subclasses of IngestPipeline need to provide a concrete implementation of
* this class that provides the additional operations that the pipeline
* requires.
*/
static abstract class PipelineModule<T extends IngestTask> implements IngestModule {
@ -369,8 +367,9 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
private volatile Date processingStartTime;
/**
* Constructs an instance of an abstract superclass for a decorator that
* adds ingest infrastructure operations to an ingest module.
* Constructs an instance of an abstract superclass for an ingest module
* decorator that adds ingest infrastructure operations to the process()
* method of an ingest module.
*
* @param module The ingest module to be wrapped.
* @param displayName The display name for the module.
@ -378,7 +377,7 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
PipelineModule(IngestModule module, String displayName) {
this.module = module;
this.displayName = displayName;
this.processingStartTime = new Date();
processingStartTime = new Date();
}
/**
@ -410,8 +409,8 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
/**
* Gets the the processing start time for the decorated module.
*
* @return The start time, will be null if the module has not started
* processing the data source yet.
* @return The start time, not valid if setProcessingStartTime() has not
* been called first.
*/
Date getProcessingStartTime() {
return new Date(processingStartTime.getTime());
@ -423,17 +422,17 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
}
/**
* Executes an ingest task using the process() method of the decorated
* Performs an ingest task using the process() method of the decorated
* module.
*
* @param ingestJobPipeline The ingest job pipeline that owns the ingest
* task pipeline this module belongs to.
* @param task The task to execute.
* @param ingestJobExecutor The ingest job executor that owns the ingest
* pipeline to which this module belongs.
* @param task The task to perform.
*
* @throws IngestModuleException Exception thrown if there is an error
* performing the task.
*/
abstract void executeTask(IngestModulePipelines ingestJobPipeline, T task) throws IngestModuleException;
abstract void process(IngestJobExecutor ingestJobExecutor, T task) throws IngestModuleException;
@Override
public void shutDown() {
@ -443,28 +442,28 @@ abstract class IngestTaskPipeline<T extends IngestTask> {
}
/**
* An exception thrown by an ingest task pipeline.
* An exception thrown by an ingest pipeline.
*/
public static class IngestTaskPipelineException extends Exception {
static class IngestPipelineException extends Exception {
private static final long serialVersionUID = 1L;
/**
* Constructs an exception to be thrown by an ingest task pipeline.
* Constructs an exception to be thrown by an ingest pipeline.
*
* @param message The exception message.
*/
public IngestTaskPipelineException(String message) {
IngestPipelineException(String message) {
super(message);
}
/**
* Constructs an exception to be thrown by an ingest task pipeline.
* Constructs an exception to be thrown by an ingest pipeline.
*
* @param message The exception message.
* @param cause The exception cause.
*/
public IngestTaskPipelineException(String message, Throwable cause) {
IngestPipelineException(String message, Throwable cause) {
super(message, cause);
}

6
Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java
View File

@ -29,7 +29,7 @@ import org.sleuthkit.datamodel.Content;
abstract class IngestTask {
private final static long NOT_SET = Long.MIN_VALUE;
private final IngestModulePipelines ingestJobPipeline;
private final IngestJobExecutor ingestJobPipeline;
private long threadId;
/**
@ -41,7 +41,7 @@ abstract class IngestTask {
* @param ingestJobPipeline The ingest job pipeline to use to execute the
* task.
*/
IngestTask(IngestModulePipelines ingestJobPipeline) {
IngestTask(IngestJobExecutor ingestJobPipeline) {
this.ingestJobPipeline = ingestJobPipeline;
threadId = NOT_SET;
}
@ -51,7 +51,7 @@ abstract class IngestTask {
*
* @return The ingest job pipeline.
*/
IngestModulePipelines getIngestJobPipeline() {
IngestJobExecutor getIngestJobPipeline() {
return ingestJobPipeline;
}

18
Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java
View File

@ -138,7 +138,7 @@ final class IngestTasksScheduler {
* task to the pipeline for processing by the
* pipeline's ingest modules.
*/
synchronized void scheduleIngestTasks(IngestModulePipelines ingestPipeline) {
synchronized void scheduleIngestTasks(IngestJobExecutor ingestPipeline) {
if (!ingestPipeline.isCancelled()) {
if (ingestPipeline.hasDataSourceIngestModules()) {
scheduleDataSourceIngestTask(ingestPipeline);
@ -163,7 +163,7 @@ final class IngestTasksScheduler {
* task to the pipeline for processing by the
* pipeline's ingest modules.
*/
synchronized void scheduleDataSourceIngestTask(IngestModulePipelines ingestPipeline) {
synchronized void scheduleDataSourceIngestTask(IngestJobExecutor ingestPipeline) {
if (!ingestPipeline.isCancelled()) {
DataSourceIngestTask task = new DataSourceIngestTask(ingestPipeline);
try {
@ -190,7 +190,7 @@ final class IngestTasksScheduler {
* empty, then all if the files from the data source
* are candidates for scheduling.
*/
synchronized void scheduleFileIngestTasks(IngestModulePipelines ingestPipeline, Collection<AbstractFile> files) {
synchronized void scheduleFileIngestTasks(IngestJobExecutor ingestPipeline, Collection<AbstractFile> files) {
if (!ingestPipeline.isCancelled()) {
Collection<AbstractFile> candidateFiles;
if (files.isEmpty()) {
@ -220,7 +220,7 @@ final class IngestTasksScheduler {
* processing by the pipeline's ingest modules.
* @param files A list of file object IDs for the streamed files.
*/
synchronized void scheduleStreamedFileIngestTasks(IngestModulePipelines ingestPipeline, List<Long> fileIds) {
synchronized void scheduleStreamedFileIngestTasks(IngestJobExecutor ingestPipeline, List<Long> fileIds) {
if (!ingestPipeline.isCancelled()) {
for (long id : fileIds) {
/*
@ -252,7 +252,7 @@ final class IngestTasksScheduler {
* processing by the pipeline's ingest modules.
* @param files The files.
*/
synchronized void fastTrackFileIngestTasks(IngestModulePipelines ingestPipeline, Collection<AbstractFile> files) {
synchronized void fastTrackFileIngestTasks(IngestJobExecutor ingestPipeline, Collection<AbstractFile> files) {
if (!ingestPipeline.isCancelled()) {
/*
* Put the files directly into the queue for the file ingest
@ -290,7 +290,7 @@ final class IngestTasksScheduler {
* target Content of the task to the pipeline for
* processing by the pipeline's ingest modules.
*/
synchronized void scheduleDataArtifactIngestTasks(IngestModulePipelines ingestPipeline) {
synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor ingestPipeline) {
if (!ingestPipeline.isCancelled()) {
Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard();
try {
@ -318,7 +318,7 @@ final class IngestTasksScheduler {
* source; if empty, then all of the data artifacts
* from the data source will be scheduled.
*/
synchronized void scheduleDataArtifactIngestTasks(IngestModulePipelines ingestPipeline, List<DataArtifact> artifacts) {
synchronized void scheduleDataArtifactIngestTasks(IngestJobExecutor ingestPipeline, List<DataArtifact> artifacts) {
if (!ingestPipeline.isCancelled()) {
for (DataArtifact artifact : artifacts) {
DataArtifactIngestTask task = new DataArtifactIngestTask(ingestPipeline, artifact);
@ -373,7 +373,7 @@ final class IngestTasksScheduler {
*
* @return True or false.
*/
synchronized boolean currentTasksAreCompleted(IngestModulePipelines ingestPipeline) {
synchronized boolean currentTasksAreCompleted(IngestJobExecutor ingestPipeline) {
long pipelineId = ingestPipeline.getIngestJobId();
return !(dataSourceIngestTasksQueue.hasTasksForJob(pipelineId)
|| hasTasksForJob(topLevelFileIngestTasksQueue, pipelineId)
@ -402,7 +402,7 @@ final class IngestTasksScheduler {
*
* @param ingestJobPipeline The ingest pipeline for the job.
*/
synchronized void cancelPendingFileTasksForIngestJob(IngestModulePipelines ingestJobPipeline) {
synchronized void cancelPendingFileTasksForIngestJob(IngestJobExecutor ingestJobPipeline) {
long jobId = ingestJobPipeline.getIngestJobId();
removeTasksForJob(topLevelFileIngestTasksQueue, jobId);
removeTasksForJob(batchedFileIngestTasksQueue, jobId);

3
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java
View File

@ -1820,7 +1820,6 @@ class ExtractRegistry extends Extract {
*/
void createShellBagArtifacts(AbstractFile regFile, List<ShellBag> shellbags) throws TskCoreException {
List<BlackboardArtifact> artifacts = new ArrayList<>();
List<DataArtifact> dataArtifacts = new ArrayList<>();
try {
for (ShellBag bag : shellbags) {
Collection<BlackboardAttribute> attributes = new ArrayList<>();
@ -1850,12 +1849,10 @@ class ExtractRegistry extends Extract {
BlackboardArtifact artifact = createArtifactWithAttributes(getShellBagArtifact(), regFile, attributes);
artifacts.add(artifact);
dataArtifacts.add((DataArtifact)artifact);
}
} finally {
if(!context.dataSourceIngestIsCancelled()) {
postArtifacts(artifacts);
context.addDataArtifactsToJob(dataArtifacts);
}
}
}