mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-14 17:06:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into 7279-updateOSAccountTreeToRefresh

Browse Source
This commit is contained in:
Kelly Kelly 2021-02-24 10:38:11 -05:00
commit decb76b860
13 changed files with 550 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java
View File

@ -162,8 +162,7 @@ final class ArtifactsListPanel extends AbstractArtifactListPanel {
@ThreadConfined(type = ThreadConfined.ThreadType.AWT)
@Override
void clearList() {
tableModel.setContents(new ArrayList<>());
tableModel.fireTableDataChanged();
addArtifacts(new ArrayList<>());
}
/**

8
Core/src/org/sleuthkit/autopsy/discovery/ui/DomainDetailsPanel.java
View File

@ -80,9 +80,9 @@ final class DomainDetailsPanel extends JPanel {
if (selectedTabName == null || !selectedTabName.equals(newTabTitle)) {
selectedTabName = newTabTitle;
Component selectedComponent = jTabbedPane1.getSelectedComponent();
if (selectedComponent instanceof DomainArtifactsTabPanel) {
if (!StringUtils.isBlank(domain) && selectedComponent instanceof DomainArtifactsTabPanel) {
runDomainWorker((DomainArtifactsTabPanel) selectedComponent, true);
} else if (selectedComponent instanceof MiniTimelinePanel) {
} else if (!StringUtils.isBlank(domain) && selectedComponent instanceof MiniTimelinePanel) {
runMiniTimelineWorker((MiniTimelinePanel) selectedComponent, true);
}
}
@ -170,13 +170,13 @@ final class DomainDetailsPanel extends JPanel {
@Subscribe
void handlePopulateDomainTabsEvent(DiscoveryEventUtils.PopulateDomainTabsEvent populateEvent) {
SwingUtilities.invokeLater(() -> {
if (StringUtils.isBlank(populateEvent.getDomain())) {
domain = populateEvent.getDomain();
if (StringUtils.isBlank(domain)) {
resetTabsStatus();
//send fade out event
DiscoveryEventUtils.getDiscoveryEventBus().post(new DiscoveryEventUtils.DetailsVisibleEvent(false));
} else {
resetTabsStatus();
domain = populateEvent.getDomain();
Component selectedComponent = jTabbedPane1.getSelectedComponent();
if (selectedComponent instanceof DomainArtifactsTabPanel) {
runDomainWorker((DomainArtifactsTabPanel) selectedComponent, false);

5
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java
View File

@ -23,6 +23,11 @@ import org.sleuthkit.datamodel.Content;
/**
* An adapter that provides a no-op implementation of the startUp() method for
* data source ingest modules.
*
* NOTE: As of Java 8, interfaces can have default methods.
* DataSourceIngestModule now provides default no-op versions of startUp() and
* shutDown(). This class is no longer needed and can be deprecated when
* convenient.
*/
public abstract class DataSourceIngestModuleAdapter implements DataSourceIngestModule {

198
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java
View File

@ -1,15 +1,15 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2016 Basis Technology Corp.
*
* Copyright 2014-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -18,184 +18,86 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.Content;
/**
* This class manages a sequence of data source level ingest modules for an
* ingestJobPipeline. It starts the modules, runs data sources through them, and
* shuts them down when data source level ingest is complete.
* <p>
* This class is thread-safe.
* A pipeline of data source level ingest modules for performing data source
* level ingest tasks for an ingest job.
*/
final class DataSourceIngestPipeline {
final class DataSourceIngestPipeline extends IngestTaskPipeline<DataSourceIngestTask> {
private static final IngestManager ingestManager = IngestManager.getInstance();
private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName());
private final IngestJobPipeline ingestJobPipeline;
private final List<PipelineModule> modules = new ArrayList<>();
private volatile PipelineModule currentModule;
private static final IngestManager ingestManager = IngestManager.getInstance();
/**
* Constructs an object that manages a sequence of data source level ingest
* modules. It starts the modules, runs data sources through them, and shuts
* them down when data source level ingest is complete.
* Constructs a pipeline of data source level ingest modules for performing
* data source level ingest tasks for an ingest job.
*
* @param ingestJobPipeline The ingestJobPipeline that owns this pipeline.
* @param moduleTemplates Templates for the creating the ingest modules that
* make up this pipeline.
* @param ingestJobPipeline The ingest job pipeline that owns this pipeline.
* @param moduleTemplates The ingest module templates that define this
* pipeline.
*/
DataSourceIngestPipeline(IngestJobPipeline ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
this.ingestJobPipeline = ingestJobPipeline;
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) {
PipelineModule module = new PipelineModule(template.createDataSourceIngestModule(), template.getModuleName());
modules.add(module);
}
}
super(ingestJobPipeline, moduleTemplates);
}
/**
* Indicates whether or not there are any ingest modules in this pipeline.
*
* @return True or false.
*/
boolean isEmpty() {
return modules.isEmpty();
@Override
Optional<IngestTaskPipeline.PipelineModule<DataSourceIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
Optional<IngestTaskPipeline.PipelineModule<DataSourceIngestTask>> module = Optional.empty();
if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModule ingestModule = template.createDataSourceIngestModule();
module = Optional.of(new DataSourcePipelineModule(ingestModule, template.getModuleName()));
}
return module;
}
/**
* Starts up the ingest modules in this pipeline.
*
* @return A list of ingest module startup errors, possibly empty.
*/
synchronized List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
for (PipelineModule module : modules) {
try {
module.startUp(new IngestJobContext(this.ingestJobPipeline));
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
return errors;
@Override
void prepareTask(DataSourceIngestTask task) {
}
/**
* Runs a data source through the ingest modules in sequential order.
*
* @param task A data source level ingest task containing a data source to
* be processed.
*
* @return A list of processing errors, possible empty.
*/
synchronized List<IngestModuleError> process(DataSourceIngestTask task) {
List<IngestModuleError> errors = new ArrayList<>();
if (!this.ingestJobPipeline.isCancelled()) {
Content dataSource = task.getDataSource();
for (PipelineModule module : modules) {
try {
this.currentModule = module;
String displayName = NbBundle.getMessage(this.getClass(),
"IngestJob.progress.dataSourceIngest.displayName",
module.getDisplayName(), dataSource.getName());
this.ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(displayName);
this.ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate();
DataSourceIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName());
logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) starting", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS
module.process(dataSource, new DataSourceIngestModuleProgress(this.ingestJobPipeline));
logger.log(Level.INFO, "{0} analysis of {1} (pipeline={2}) finished", new Object[]{module.getDisplayName(), ingestJobPipeline.getDataSource().getName(), ingestJobPipeline.getId()}); //NON-NLS
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (this.ingestJobPipeline.isCancelled()) {
break;
} else if (this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) {
this.ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(currentModule.getDisplayName());
}
}
}
this.currentModule = null;
@Override
void completeTask(DataSourceIngestTask task) {
ingestManager.setIngestTaskProgressCompleted(task);
return errors;
}
/**
* Gets the currently running module.
*
* @return The module, possibly null if no module is currently running.
* A wrapper that adds ingest infrastructure operations to a data source
* level ingest module.
*/
PipelineModule getCurrentlyRunningModule() {
return this.currentModule;
}
/**
* This class decorates a data source level ingest module with a display
* name and a processing start time.
*/
static class PipelineModule implements DataSourceIngestModule {
static final class DataSourcePipelineModule extends IngestTaskPipeline.PipelineModule<DataSourceIngestTask> {
private final DataSourceIngestModule module;
private final String displayName;
private volatile Date processingStartTime;
/**
* Constructs an object that decorates a data source level ingest module
* with a display name and a processing start time.
*
* @param module The data source level ingest module to be
* decorated.
* @param displayName The display name.
* Constructs a wrapper that adds ingest infrastructure operations to a
* data source level ingest module.
*/
PipelineModule(DataSourceIngestModule module, String displayName) {
DataSourcePipelineModule(DataSourceIngestModule module, String displayName) {
super(module, displayName);
this.module = module;
this.displayName = displayName;
this.processingStartTime = new Date();
}
/**
* Gets the class name of the decorated ingest module.
*
* @return The class name.
*/
String getClassName() {
return this.module.getClass().getCanonicalName();
}
/**
* Gets the display of the decorated ingest module.
*
* @return The display name.
*/
String getDisplayName() {
return this.displayName;
}
/**
* Gets the time the decorated ingest module started processing the data
* source.
*
* @return The start time, will be null if the module has not started
* processing the data source yet.
*/
Date getProcessingStartTime() {
return this.processingStartTime;
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
this.module.startUp(context);
}
@Override
public IngestModule.ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
this.processingStartTime = new Date();
return this.module.process(dataSource, statusHelper);
void performTask(IngestJobPipeline ingestJobPipeline, DataSourceIngestTask task) throws IngestModuleException {
Content dataSource = task.getDataSource();
String progressBarDisplayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", getDisplayName(), dataSource.getName());
ingestJobPipeline.updateDataSourceIngestProgressBarDisplayName(progressBarDisplayName);
ingestJobPipeline.switchDataSourceIngestProgressBarToIndeterminate();
ingestManager.setIngestTaskProgress(task, getDisplayName());
logger.log(Level.INFO, "{0} analysis of {1} starting", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS
ProcessResult result = module.process(dataSource, new DataSourceIngestModuleProgress(ingestJobPipeline));
logger.log(Level.INFO, "{0} analysis of {1} finished", new Object[]{getDisplayName(), dataSource.getName()}); //NON-NLS
if (!ingestJobPipeline.isCancelled() && ingestJobPipeline.currentDataSourceIngestModuleIsCancelled()) {
ingestJobPipeline.currentDataSourceIngestModuleCancellationCompleted(getDisplayName());
}
if (result == ProcessResult.ERROR) {
throw new IngestModuleException(String.format("%s experienced an error analyzing %s (data source objId = %d)", getDisplayName(), dataSource.getName(), dataSource.getId())); //NON-NLS
}
}
}

11
Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Copyright 2014-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -36,13 +36,4 @@ public interface FileIngestModule extends IngestModule {
*/
ProcessResult process(AbstractFile file);
/**
* Invoked by Autopsy when an ingest job is completed (either because the
* data has been analyzed or because the job was canceled - check
* IngestJobContext.fileIngestIsCancelled()), before the ingest module
* instance is discarded. The module should respond by doing things like
* releasing private resources, submitting final results, and posting a
* final ingest message.
*/
void shutDown();
}

4
Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java
View File

@ -23,6 +23,10 @@ import org.sleuthkit.datamodel.AbstractFile;
/**
* An adapter that provides no-op implementations of the startUp() and
* shutDown() methods for file ingest modules.
*
* NOTE: As of Java 8, interfaces can have default methods. FileIngestModule now
* provides default no-op versions of startUp() and shutDown(). This class is no
* longer needed and can be deprecated when convenient.
*/
public abstract class FileIngestModuleAdapter implements FileIngestModule {

250
Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java
View File

@ -1,15 +1,15 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014-2015 Basis Technology Corp.
*
* Copyright 2014-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -18,223 +18,109 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import java.util.Optional;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
/**
* This class manages a sequence of file level ingest modules for an
* ingest job pipeline. It starts the modules, runs files through them, and shuts them
* down when file level ingest is complete.
* <p>
* This class is thread-safe.
* A pipeline of file ingest modules for performing file ingest tasks for an
* ingest job.
*/
final class FileIngestPipeline {
final class FileIngestPipeline extends IngestTaskPipeline<FileIngestTask> {
private static final IngestManager ingestManager = IngestManager.getInstance();
private final IngestJobPipeline ingestJobPipeline;
private final List<PipelineModule> modules = new ArrayList<>();
private Date startTime;
private volatile boolean running;
/**
* Constructs an object that manages a sequence of file level ingest
* modules. It starts the modules, runs files through them, and shuts them
* down when file level ingest is complete.
* Constructs a pipeline of file ingest modules for performing file ingest
* tasks for an ingest job.
*
* @param ingestJobPipeline The ingestJobPipeline that owns the pipeline.
* @param moduleTemplates The ingest module templates that define the
* pipeline.
* @param ingestJobPipeline The ingest job pipeline that owns this pipeline.
* @param moduleTemplates The ingest module templates that define this
* pipeline.
*/
FileIngestPipeline(IngestJobPipeline ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
super(ingestJobPipeline, moduleTemplates);
this.ingestJobPipeline = ingestJobPipeline;
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isFileIngestModuleTemplate()) {
PipelineModule module = new PipelineModule(template.createFileIngestModule(), template.getModuleName());
modules.add(module);
}
}
@Override
Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> acceptModuleTemplate(IngestModuleTemplate template) {
Optional<IngestTaskPipeline.PipelineModule<FileIngestTask>> module = Optional.empty();
if (template.isFileIngestModuleTemplate()) {
FileIngestModule ingestModule = template.createFileIngestModule();
module = Optional.of(new FileIngestPipelineModule(ingestModule, template.getModuleName()));
}
return module;
}
/**
* Queries whether or not there are any ingest modules in this pipeline.
*
* @return True or false.
*/
boolean isEmpty() {
return this.modules.isEmpty();
@Override
void prepareTask(FileIngestTask task) throws IngestTaskPipelineException {
}
/**
* Queries whether or not this pipeline is running.
*
* @return True or false.
*/
boolean isRunning() {
return this.running;
}
/**
* Returns the start up time of this pipeline.
*
* @return The file processing start time, may be null if this pipeline has
* not been started yet.
*/
Date getStartTime() {
return this.startTime;
}
/**
* Starts up all of the ingest modules in the pipeline.
*
* @return List of start up errors, possibly empty.
*/
synchronized List<IngestModuleError> startUp() {
this.startTime = new Date();
this.running = true;
List<IngestModuleError> errors = new ArrayList<>();
for (PipelineModule module : this.modules) {
try {
module.startUp(new IngestJobContext(this.ingestJobPipeline));
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
@Override
void completeTask(FileIngestTask task) throws IngestTaskPipelineException {
AbstractFile file = null;
try {
file = task.getFile();
} catch (TskCoreException ex) {
throw new IngestTaskPipelineException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
}
return errors;
}
/**
* Runs a file through the ingest modules in sequential order.
*
* @param task A file level ingest task containing a file to be processed.
*
* @return A list of processing errors, possible empty.
*/
synchronized List<IngestModuleError> process(FileIngestTask task) {
List<IngestModuleError> errors = new ArrayList<>();
if (!this.ingestJobPipeline.isCancelled()) {
AbstractFile file;
try {
file = task.getFile();
} catch (TskCoreException ex) {
// In practice, this task would never have been enqueued since the file
// lookup would have failed there.
errors.add(new IngestModuleError("File Ingest Pipeline", ex)); // NON-NLS
FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task);
return errors;
try {
if (!ingestJobPipeline.isCancelled()) {
/*
* Save any updates from the ingest modules to the case
* database.
*/
file.save();
}
for (PipelineModule module : this.modules) {
try {
FileIngestPipeline.ingestManager.setIngestTaskProgress(task, module.getDisplayName());
this.ingestJobPipeline.setCurrentFileIngestModule(module.getDisplayName(), task.getFile().getName());
module.process(file);
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (this.ingestJobPipeline.isCancelled()) {
break;
}
}
if (!this.ingestJobPipeline.isCancelled()) {
// Save any properties that have not already been saved to the database
try{
file.save();
} catch (TskCoreException ex){
Logger.getLogger(FileIngestPipeline.class.getName()).log(Level.SEVERE, "Failed to save data for file " + file.getId(), ex); //NON-NLS
}
} catch (TskCoreException ex) {
throw new IngestTaskPipelineException(String.format("Failed to save updated data for file (file objId = %d)", task.getFileId()), ex); //NON-NLS
} finally {
if (!ingestJobPipeline.isCancelled()) {
IngestManager.getInstance().fireFileIngestDone(file);
}
file.close();
ingestManager.setIngestTaskProgressCompleted(task);
}
FileIngestPipeline.ingestManager.setIngestTaskProgressCompleted(task);
return errors;
}
/**
* Shuts down all of the modules in the pipeline.
*
* @return A list of shut down errors, possibly empty.
* A wrapper that adds ingest infrastructure operations to a file ingest
* module.
*/
synchronized List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>();
if (this.running == true) { // Don't shut down pipelines that never started
for (PipelineModule module : this.modules) {
try {
module.shutDown();
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
String msg = ex.getMessage();
// Jython run-time errors don't seem to have a message, but have details in toString.
if (msg == null) {
msg = ex.toString();
}
MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg);
}
}
}
this.running = false;
return errors;
}
/**
* This class decorates a file level ingest module with a display name.
*/
private static final class PipelineModule implements FileIngestModule {
static final class FileIngestPipelineModule extends IngestTaskPipeline.PipelineModule<FileIngestTask> {
private final FileIngestModule module;
private final String displayName;
/**
* Constructs an object that decorates a file level ingest module with a
* display name.
* Constructs a wrapper that adds ingest infrastructure operations to a
* file ingest module.
*
* @param module The file level ingest module to be decorated.
* @param displayName The display name.
*
* @param module The module.
* @param displayName The display name of the module.
*/
PipelineModule(FileIngestModule module, String displayName) {
FileIngestPipelineModule(FileIngestModule module, String displayName) {
super(module, displayName);
this.module = module;
this.displayName = displayName;
}
/**
* Gets the class name of the decorated ingest module.
*
* @return The class name.
*/
String getClassName() {
return module.getClass().getCanonicalName();
}
/**
* Gets the display name of the decorated ingest module.
*
* @return The display name.
*/
String getDisplayName() {
return displayName;
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
module.startUp(context);
}
@Override
public IngestModule.ProcessResult process(AbstractFile file) {
return module.process(file);
}
@Override
public void shutDown() {
module.shutDown();
void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException {
AbstractFile file = null;
try {
file = task.getFile();
} catch (TskCoreException ex) {
throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS
}
ingestManager.setIngestTaskProgress(task, getDisplayName());
ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName());
ProcessResult result = module.process(file);
if (result == ProcessResult.ERROR) {
throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS
}
}
}

6
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -416,7 +416,7 @@ public final class IngestJob {
Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot);
dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot));
if (null == dataSourceModule) {
DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule();
DataSourceIngestPipeline.DataSourcePipelineModule module = snapshot.getDataSourceLevelIngestModule();
if (null != module) {
dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module);
}
@ -500,7 +500,7 @@ public final class IngestJob {
public static class DataSourceIngestModuleHandle {
private final IngestJobPipeline ingestJobPipeline;
private final DataSourceIngestPipeline.PipelineModule module;
private final DataSourceIngestPipeline.DataSourcePipelineModule module;
private final boolean cancelled;
/**
@ -511,7 +511,7 @@ public final class IngestJob {
* @param ingestJobPipeline The ingestJobPipeline that owns the data source level ingest module.
* @param module The data source level ingest module.
*/
private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.PipelineModule module) {
private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.DataSourcePipelineModule module) {
this.ingestJobPipeline = ingestJobPipeline;
this.module = module;
this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled();

22
Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014-2019 Basis Technology Corp.
* Copyright 2014-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -553,7 +553,7 @@ final class IngestJobPipeline {
/*
* If the data-source-level ingest pipelines were successfully started,
* start the Start the file-level ingest pipelines (one per file ingest
* start the file-level ingest pipelines (one per pipeline file ingest
* thread).
*/
if (errors.isEmpty()) {
@ -940,7 +940,7 @@ final class IngestJobPipeline {
synchronized (this.dataSourceIngestPipelineLock) {
if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) {
List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(this.currentDataSourceIngestPipeline.process(task));
errors.addAll(this.currentDataSourceIngestPipeline.performTask(task));
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
@ -1014,7 +1014,7 @@ final class IngestJobPipeline {
* Run the file through the pipeline.
*/
List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(pipeline.process(task));
errors.addAll(pipeline.performTask(task));
if (!errors.isEmpty()) {
logIngestModuleErrors(errors, file);
}
@ -1232,9 +1232,9 @@ final class IngestJobPipeline {
*
* @return The currently running module, may be null.
*/
DataSourceIngestPipeline.PipelineModule getCurrentDataSourceIngestModule() {
if (null != this.currentDataSourceIngestPipeline) {
return this.currentDataSourceIngestPipeline.getCurrentlyRunningModule();
DataSourceIngestPipeline.DataSourcePipelineModule getCurrentDataSourceIngestModule() {
if (null != currentDataSourceIngestPipeline) {
return (DataSourceIngestPipeline.DataSourcePipelineModule) currentDataSourceIngestPipeline.getCurrentlyRunningModule();
} else {
return null;
}
@ -1274,7 +1274,7 @@ final class IngestJobPipeline {
}
}
}
// If a data source had no tasks in progress it may now be complete.
checkForStageCompleted();
}
@ -1353,18 +1353,18 @@ final class IngestJobPipeline {
logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis", error.getModuleDisplayName()), error.getThrowable()); //NON-NLS
}
}
/**
* Write ingest module errors to the log.
*
* @param errors The errors.
* @param file AbstractFile that caused the errors.
* @param file AbstractFile that caused the errors.
*/
private void logIngestModuleErrors(List<IngestModuleError> errors, AbstractFile file) {
for (IngestModuleError error : errors) {
logErrorMessage(Level.SEVERE, String.format("%s experienced an error during analysis while processing file %s, object ID %d", error.getModuleDisplayName(), file.getName(), file.getId()), error.getThrowable()); //NON-NLS
}
}
}
/**
* Gets a snapshot of this jobs state and performance.

2
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2019 Basis Technology Corp.
* Copyright 2012-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");

86
Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Copyright 2014-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -25,9 +25,8 @@ package org.sleuthkit.autopsy.ingest;
* ingest job it performs (one for each thread that it is using).
*
* Autopsy will call startUp() before any data is processed, will pass any data
* to be analyzed into the process() method (FileIngestModule.process() or
* DataSourceIngestModule.process()), and call shutDown() after either all data
* is analyzed or the user has canceled the job.
* to be analyzed into the process() method, and call shutDown() after either
* all data is analyzed or the user has canceled the job.
*
* Autopsy may use multiple threads to complete an ingest job, but it is
* guaranteed that a module instance will always be called from a single thread.
@ -47,25 +46,53 @@ package org.sleuthkit.autopsy.ingest;
public interface IngestModule {
/**
* A return code for derived class process() methods.
* Invoked by Autopsy to allow an ingest module instance to set up any
* internal data structures and acquire any private resources it will need
* during an ingest job. If the module depends on loading any resources, it
* should do so in this method so that it can throw an exception in the case
* of an error and alert the user. Exceptions that are thrown from startUp()
* are logged and stop processing of the data source.
*
* IMPORTANT: If the module instances must share resources, the modules are
* responsible for synchronizing access to the shared resources and doing
* reference counting as required to release those resources correctly.
* Also, more than one ingest job may be in progress at any given time. This
* must also be taken into consideration when sharing resources between
* module instances. See IngestModuleReferenceCounter.
*
* @param context Provides data and services specific to the ingest job and
* the ingest pipeline of which the module is a part.
*
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
*/
public enum ProcessResult {
OK,
ERROR
};
default void startUp(IngestJobContext context) throws IngestModuleException {
}
/**
* A custom exception for the use of ingest modules.
* Invoked by Autopsy when an ingest job is completed (either because the
* data has been analyzed or because the job was cancelled), before the
* ingest module instance is discarded. The module should respond by doing
* things like releasing private resources, submitting final results, and
* posting a final ingest message.
*
* IMPORTANT: If the module instances must share resources, the modules are
* responsible for synchronizing access to the shared resources and doing
* reference counting as required to release those resources correctly.
* Also, more than one ingest job may be in progress at any given time. This
* must also be taken into consideration when sharing resources between
* module instances. See IngestModuleReferenceCounter.
*
*/
default void shutDown() {
}
/**
* An exception for the use of ingest modules.
*/
public class IngestModuleException extends Exception {
private static final long serialVersionUID = 1L;
@Deprecated
public IngestModuleException() {
}
public IngestModuleException(String message) {
super(message);
}
@ -73,26 +100,21 @@ public interface IngestModule {
public IngestModuleException(String message, Throwable cause) {
super(message, cause);
}
@Deprecated
public IngestModuleException() {
}
}
/**
* Invoked by Autopsy to allow an ingest module instance to set up any
* internal data structures and acquire any private resources it will need
* during an ingest job. If the module depends on loading any resources, it
* should do so in this method so that it can throw an exception in the case
* of an error and alert the user. Exceptions that are thrown from process()
* and shutDown() are logged, but do not stop processing of the data source.
*
* @param context Provides data and services specific to the ingest job and
* the ingest pipeline of which the module is a part.
*
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
* A return code for subclass process() methods.
*/
void startUp(IngestJobContext context) throws IngestModuleException;
public enum ProcessResult {
OK,
ERROR
};
/**
* TODO: The next time an API change is legal, add a cancel() method and
* remove the "ingest job is canceled" queries from the IngestJobContext
* class.
*/
}

345
Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java Executable file
View File

@ -0,0 +1,345 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
/**
* An abstract superclass for pipelines of ingest modules for a given ingest
* task type. Some examples of ingest task types: data source level ingest
* tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses need to
* implement a specialization of the inner PipelineModule abstract superclass
* for the type of ingest modules that make up the pipeline.
*
* @param <T> The ingest task type.
*/
abstract class IngestTaskPipeline<T extends IngestTask> {
private static final IngestManager ingestManager = IngestManager.getInstance();
private final IngestJobPipeline ingestJobPipeline;
private final List<IngestModuleTemplate> moduleTemplates;
private final List<PipelineModule<T>> modules;
private volatile Date startTime;
private volatile boolean running;
private volatile PipelineModule<T> currentModule;
/**
* Constructs an instance of an abstract superclass for pipelines of ingest
* modules for a given ingest task type. Some examples of ingest task types:
* data source level ingest tasks, file ingest tasks, data artifact ingest
* tasks, etc. Subclasses need to implement a specialization of the inner
* PipelineModule abstract superclass for the type of ingest modules that
* make up the pipeline.
*
* @param ingestJobPipeline The ingest job pipeline that owns this pipeline.
* @param moduleTemplates The ingest module templates that define this
* pipeline.
*/
IngestTaskPipeline(IngestJobPipeline ingestJobPipeline, List<IngestModuleTemplate> moduleTemplates) {
this.ingestJobPipeline = ingestJobPipeline;
this.moduleTemplates = moduleTemplates;
modules = new ArrayList<>();
}
/**
* Indicates whether or not there are any ingest modules in this pipeline.
*
* @return True or false.
*/
boolean isEmpty() {
return modules.isEmpty();
}
/**
* Queries whether or not this pipeline is running, i.e., started and not
* shut down.
*
* @return True or false.
*/
boolean isRunning() {
return running;
}
/**
* Starts up the ingest modules in this pipeline.
*
* @return A list of ingest module startup errors, possibly empty.
*/
List<IngestModuleError> startUp() {
createIngestModules(moduleTemplates);
return startUpIngestModules();
}
/**
* Creates the ingest modules for this pipeline.
*
* @param moduleTemplates The ingest module templates avaialble to this
* pipeline.
*/
private void createIngestModules(List<IngestModuleTemplate> moduleTemplates) {
for (IngestModuleTemplate template : moduleTemplates) {
Optional<PipelineModule<T>> module = acceptModuleTemplate(template);
if (module.isPresent()) {
modules.add(module.get());
}
}
}
/**
* Determines if the type of ingest module that can be created from a given
* ingest module template should be added to this pipeline. If so, the
* ingest module is created and returned.
*
* @param ingestModuleTemplate The ingest module template to be used or
* ignored, as appropriate to the pipeline type.
*
* @return An Optional that is either empty or contains a newly created and
* wrapped ingest module.
*/
abstract Optional<PipelineModule<T>> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate);
/**
* Starts up the ingest modules in the pipeline.
*
* @return A list of ingest module startup errors, possibly empty.
*/
private List<IngestModuleError> startUpIngestModules() {
startTime = new Date();
running = true;
List<IngestModuleError> errors = new ArrayList<>();
for (PipelineModule<T> module : modules) {
try {
module.startUp(new IngestJobContext(ingestJobPipeline));
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
return errors;
}
/**
* Returns the start up time of this pipeline.
*
* @return The file processing start time, may be null if this pipeline has
* not been started yet.
*/
Date getStartTime() {
if (startTime == null) {
throw new IllegalStateException("startUp() was not called"); //NON-NLS
}
return new Date(startTime.getTime());
}
/**
* Does any preparation required before performing a task.
*
* @param task The task.
*
* @throws IngestTaskPipelineException Thrown if there is an error preparing
* to perform the task.
*/
abstract void prepareTask(T task) throws IngestTaskPipelineException;
/**
* Performs an ingest task using the ingest modules in this pipeline.
*
* @param task The task.
*
* @return A list of ingest module task processing errors, possibly empty.
*/
List<IngestModuleError> performTask(T task) {
List<IngestModuleError> errors = new ArrayList<>();
if (!this.ingestJobPipeline.isCancelled()) {
try {
prepareTask(task);
} catch (IngestTaskPipelineException ex) {
errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS
return errors;
}
for (PipelineModule<T> module : modules) {
try {
currentModule = module;
currentModule.setProcessingStartTime();
module.performTask(ingestJobPipeline, task);
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (ingestJobPipeline.isCancelled()) {
break;
}
}
}
try {
completeTask(task);
} catch (IngestTaskPipelineException ex) {
errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS
}
currentModule = null;
return errors;
}
/**
* Gets the currently running module.
*
* @return The module, possibly null if no module is currently running.
*/
PipelineModule<T> getCurrentlyRunningModule() {
return currentModule;
}
/**
* Does any clean up required after performing a task.
*
* @param task The task.
*
* @throws IngestTaskPipelineException Thrown if there is an error cleaning
* up after performing the task.
*/
abstract void completeTask(T task) throws IngestTaskPipelineException;
/**
* Shuts down all of the modules in the pipeline.
*
* @return A list of shut down errors, possibly empty.
*/
List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>();
if (running == true) {
for (PipelineModule<T> module : modules) {
try {
module.shutDown();
} catch (Throwable ex) { // Catch-all exception firewall
errors.add(new IngestModuleError(module.getDisplayName(), ex));
String msg = ex.getMessage();
if (msg == null) {
/*
* Jython run-time errors don't seem to have a message,
* but have details in the string returned by
* toString().
*/
msg = ex.toString();
}
MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "FileIngestPipeline.moduleError.title.text", module.getDisplayName()), msg);
}
}
}
running = false;
return errors;
}
/**
* An abstract superclass for a wrapper that adds ingest infrastructure
* operations to an ingest module.
*/
static abstract class PipelineModule<T extends IngestTask> implements IngestModule {
private final IngestModule module;
private final String displayName;
private volatile Date processingStartTime;
/**
* Constructs an instance of an abstract superclass for a wrapper that
* adds ingest infrastructure operations to an ingest module.
*
* @param module The ingest module to be wrapped.
* @param displayName The display name for the module.
*/
PipelineModule(IngestModule module, String displayName) {
this.module = module;
this.displayName = displayName;
this.processingStartTime = new Date();
}
/**
* Gets the class name of the wrapped ingest module.
*
* @return The class name.
*/
String getClassName() {
return module.getClass().getCanonicalName();
}
/**
* Gets the display name of the wrapped ingest module.
*
* @return The display name.
*/
String getDisplayName() {
return displayName;
}
/**
* Sets the processing start time for the wrapped module to the system
* time when this method is called.
*/
void setProcessingStartTime() {
processingStartTime = new Date();
}
/**
* Gets the the processing start time for the wrapped module.
*
* @return The start time, will be null if the module has not started
* processing the data source yet.
*/
Date getProcessingStartTime() {
return new Date(processingStartTime.getTime());
}
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
module.startUp(context);
}
/**
* Performs an ingest task.
*
* @param ingestJobPipeline The ingest job pipeline that owns the ingest
* module pipeline this module belongs to.
* @param task The task to process.
*
* @throws IngestModuleException Excepton thrown if there is an error
* performing the task.
*/
abstract void performTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException;
}
/**
* An exception for the use of ingest task pipelines.
*/
public static class IngestTaskPipelineException extends Exception {
private static final long serialVersionUID = 1L;
public IngestTaskPipelineException(String message) {
super(message);
}
public IngestTaskPipelineException(String message, Throwable cause) {
super(message, cause);
}
}
}

6
Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java
View File

@ -34,7 +34,7 @@ public final class Snapshot implements Serializable {
private final long jobId;
private final long jobStartTime;
private final long snapShotTime;
transient private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
transient private final DataSourceIngestPipeline.DataSourcePipelineModule dataSourceLevelIngestModule;
private final boolean fileIngestRunning;
private final Date fileIngestStartTime;
private final long processedFiles;
@ -48,7 +48,7 @@ public final class Snapshot implements Serializable {
* Constructs an object to store basic diagnostic statistics for a data
* source ingest job.
*/
Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.PipelineModule dataSourceIngestModule,
Snapshot(String dataSourceName, long jobId, long jobStartTime, DataSourceIngestPipeline.DataSourcePipelineModule dataSourceIngestModule,
boolean fileIngestRunning, Date fileIngestStartTime,
boolean jobCancelled, IngestJob.CancellationReason cancellationReason, List<String> cancelledModules,
long processedFiles, long estimatedFilesToProcess,
@ -110,7 +110,7 @@ public final class Snapshot implements Serializable {
return jobStartTime;
}
DataSourceIngestPipeline.PipelineModule getDataSourceLevelIngestModule() {
DataSourceIngestPipeline.DataSourcePipelineModule getDataSourceLevelIngestModule() {
return this.dataSourceLevelIngestModule;
}