mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-14 17:06:16 +00:00
Merge branch 'develop' of https://github.com/sleuthkit/autopsy into develop
This commit is contained in:
commit
d07968b479
@ -73,6 +73,7 @@ import org.sleuthkit.datamodel.SleuthkitCase;
|
||||
import org.sleuthkit.datamodel.TskCoreException;
|
||||
import org.sleuthkit.datamodel.TskException;
|
||||
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
|
||||
|
||||
/**
|
||||
* Top component which displays something.
|
||||
*/
|
||||
@ -543,8 +544,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
// The current case has been closed. Reset the ExplorerManager.
|
||||
Node emptyNode = new AbstractNode(Children.LEAF);
|
||||
em.setRootContext(emptyNode);
|
||||
}
|
||||
else if (newValue != null) {
|
||||
} else if (newValue != null) {
|
||||
// A new case has been opened. Reset the forward and back
|
||||
// buttons. Note that a call to CoreComponentControl.openCoreWindows()
|
||||
// by the new Case object will lead to a componentOpened() call
|
||||
@ -598,7 +598,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
refreshTree(event.getArtifactType());
|
||||
}
|
||||
});
|
||||
} else if (changed.equals(IngestEvent.COMPLETED.toString())) {
|
||||
} else if (changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
|
||||
|| changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
|
||||
SwingUtilities.invokeLater(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
@ -668,8 +669,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
} catch (TskCoreException ex) {
|
||||
logger.log(Level.SEVERE, "Exception while calling Content.getUniquePath() for node: " + originNode);
|
||||
}
|
||||
}
|
||||
else if (originNode.getLookup().lookup(String.class) != null) {
|
||||
} else if (originNode.getLookup().lookup(String.class) != null) {
|
||||
displayName = originNode.getLookup().lookup(String.class);
|
||||
}
|
||||
dataResult.setPath(displayName);
|
||||
@ -752,8 +752,6 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
pcs.removePropertyChangeListener(listener);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Gets the tree on this DirectoryTreeTopComponent.
|
||||
*
|
||||
@ -881,13 +879,11 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
while (null == selectedNode && !selectedNodePath.isEmpty()) {
|
||||
try {
|
||||
selectedNode = NodeOp.findPath(em.getRootContext(), selectedNodePath.toArray(new String[0]));
|
||||
}
|
||||
catch (NodeNotFoundException ex) {
|
||||
} catch (NodeNotFoundException ex) {
|
||||
// The selected node may have been deleted (e.g., a deleted tag), so truncate the path and try again.
|
||||
if (selectedNodePath.size() > 1) {
|
||||
selectedNodePath.remove(selectedNodePath.size() - 1);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
StringBuilder nodePath = new StringBuilder();
|
||||
for (int i = 0; i < previouslySelectedNodePath.length; ++i) {
|
||||
nodePath.append(previouslySelectedNodePath[i]).append("/");
|
||||
@ -906,8 +902,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
}
|
||||
try {
|
||||
em.setExploredContextAndSelection(selectedNode, new Node[]{selectedNode});
|
||||
}
|
||||
catch (PropertyVetoException ex) {
|
||||
} catch (PropertyVetoException ex) {
|
||||
logger.log(Level.WARNING, "Property veto from ExplorerManager setting selection to " + selectedNode.getName(), ex);
|
||||
}
|
||||
}
|
||||
@ -970,8 +965,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
} catch (TskException ex) {
|
||||
logger.log(Level.WARNING, "Error retrieving attributes", ex);
|
||||
}
|
||||
} else if ( type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT) ||
|
||||
type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT) ) {
|
||||
} else if (type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)
|
||||
|| type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT)) {
|
||||
Node interestingItemsRootNode = resultsChilds.findChild(type.getLabel());
|
||||
Children interestingItemsRootChildren = interestingItemsRootNode.getChildren();
|
||||
try {
|
||||
@ -1033,8 +1028,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
|
||||
try {
|
||||
firePropertyChange(BlackboardResultViewer.FINISHED_DISPLAY_EVT, 0, 1);
|
||||
}
|
||||
catch (Exception e) {
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.SEVERE, "DirectoryTreeTopComponent listener threw exception", e);
|
||||
MessageNotifyUtil.Notify.show(NbBundle.getMessage(this.getClass(), "DirectoryTreeTopComponent.moduleErr"),
|
||||
NbBundle.getMessage(this.getClass(),
|
||||
|
@ -18,9 +18,7 @@
|
||||
*/
|
||||
package org.sleuthkit.autopsy.ingest;
|
||||
|
||||
import javax.swing.SwingWorker;
|
||||
import org.netbeans.api.progress.ProgressHandle;
|
||||
import org.sleuthkit.datamodel.Content;
|
||||
|
||||
/**
|
||||
* Used by data source ingest modules to report progress and detect data source
|
||||
@ -28,15 +26,11 @@ import org.sleuthkit.datamodel.Content;
|
||||
*/
|
||||
public class DataSourceIngestModuleStatusHelper {
|
||||
|
||||
private final SwingWorker worker;
|
||||
private final ProgressHandle progress;
|
||||
private final Content dataSource;
|
||||
private final IngestJob ingestJob;
|
||||
private final String moduleDisplayName;
|
||||
|
||||
DataSourceIngestModuleStatusHelper(SwingWorker worker, ProgressHandle progress, Content dataSource, String moduleDisplayName) {
|
||||
this.worker = worker;
|
||||
this.progress = progress;
|
||||
this.dataSource = dataSource;
|
||||
DataSourceIngestModuleStatusHelper(IngestJob ingestJob, String moduleDisplayName) {
|
||||
this.ingestJob = ingestJob;
|
||||
this.moduleDisplayName = moduleDisplayName;
|
||||
}
|
||||
|
||||
@ -48,7 +42,7 @@ public class DataSourceIngestModuleStatusHelper {
|
||||
* @return True if the task has been canceled, false otherwise.
|
||||
*/
|
||||
public boolean isIngestJobCancelled() {
|
||||
return worker.isCancelled();
|
||||
return (ingestJob.isCancelled());
|
||||
}
|
||||
|
||||
/**
|
||||
@ -60,9 +54,7 @@ public class DataSourceIngestModuleStatusHelper {
|
||||
* data source.
|
||||
*/
|
||||
public void switchToDeterminate(int workUnits) {
|
||||
if (progress != null) {
|
||||
progress.switchToDeterminate(workUnits);
|
||||
}
|
||||
ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -70,9 +62,7 @@ public class DataSourceIngestModuleStatusHelper {
|
||||
* the total work units to process the data source is unknown.
|
||||
*/
|
||||
public void switchToIndeterminate() {
|
||||
if (progress != null) {
|
||||
progress.switchToIndeterminate();
|
||||
}
|
||||
ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -82,8 +72,6 @@ public class DataSourceIngestModuleStatusHelper {
|
||||
* @param workUnits Number of work units performed so far by the module.
|
||||
*/
|
||||
public void progress(int workUnits) {
|
||||
if (progress != null) {
|
||||
progress.progress(this.moduleDisplayName, workUnits);
|
||||
}
|
||||
ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits);
|
||||
}
|
||||
}
|
@ -22,10 +22,6 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import javax.swing.SwingWorker;
|
||||
import org.netbeans.api.progress.ProgressHandle;
|
||||
import org.sleuthkit.autopsy.coreutils.Logger;
|
||||
import org.sleuthkit.datamodel.Content;
|
||||
|
||||
/**
|
||||
@ -34,7 +30,6 @@ import org.sleuthkit.datamodel.Content;
|
||||
*/
|
||||
final class DataSourceIngestPipeline {
|
||||
|
||||
private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName());
|
||||
private final IngestJob job;
|
||||
private final List<IngestModuleTemplate> moduleTemplates;
|
||||
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
|
||||
@ -59,7 +54,6 @@ final class DataSourceIngestPipeline {
|
||||
try {
|
||||
module.startUp(context);
|
||||
modulesByClass.put(module.getClassName(), module);
|
||||
IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), module.getDisplayName());
|
||||
} catch (Exception ex) {
|
||||
errors.add(new IngestModuleError(module.getDisplayName(), ex));
|
||||
}
|
||||
@ -81,13 +75,11 @@ final class DataSourceIngestPipeline {
|
||||
return errors;
|
||||
}
|
||||
|
||||
List<IngestModuleError> process(SwingWorker worker, ProgressHandle progress) {
|
||||
List<IngestModuleError> process() {
|
||||
List<IngestModuleError> errors = new ArrayList<>();
|
||||
Content dataSource = this.job.getDataSource();
|
||||
logger.log(Level.INFO, "Processing data source {0}", dataSource.getName());
|
||||
for (DataSourceIngestModuleDecorator module : this.modules) {
|
||||
try {
|
||||
module.process(dataSource, new DataSourceIngestModuleStatusHelper(worker, progress, dataSource, module.getDisplayName()));
|
||||
module.process(job.getDataSource(), new DataSourceIngestModuleStatusHelper(job, module.getDisplayName()));
|
||||
} catch (Exception ex) {
|
||||
errors.add(new IngestModuleError(module.getDisplayName(), ex));
|
||||
}
|
||||
@ -105,8 +97,6 @@ final class DataSourceIngestPipeline {
|
||||
module.shutDown(ingestJobCancelled);
|
||||
} catch (Exception ex) {
|
||||
errors.add(new IngestModuleError(module.getDisplayName(), ex));
|
||||
} finally {
|
||||
IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName());
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
|
@ -22,10 +22,7 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import org.sleuthkit.autopsy.coreutils.Logger;
|
||||
import org.sleuthkit.datamodel.AbstractFile;
|
||||
import org.sleuthkit.datamodel.Content;
|
||||
|
||||
/**
|
||||
* A file ingest pipeline composed of a sequence of file ingest modules
|
||||
@ -33,7 +30,6 @@ import org.sleuthkit.datamodel.Content;
|
||||
*/
|
||||
final class FileIngestPipeline {
|
||||
|
||||
private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName());
|
||||
private final IngestJob job;
|
||||
private final List<IngestModuleTemplate> moduleTemplates;
|
||||
private List<FileIngestModuleDecorator> modules = new ArrayList<>();
|
||||
@ -58,7 +54,6 @@ final class FileIngestPipeline {
|
||||
try {
|
||||
module.startUp(context);
|
||||
modulesByClass.put(module.getClassName(), module);
|
||||
IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), template.getModuleName());
|
||||
} catch (Exception ex) {
|
||||
errors.add(new IngestModuleError(module.getDisplayName(), ex));
|
||||
}
|
||||
@ -93,7 +88,9 @@ final class FileIngestPipeline {
|
||||
}
|
||||
}
|
||||
file.close();
|
||||
IngestManager.fireFileDone(file.getId());
|
||||
if (!job.isCancelled()) {
|
||||
IngestManager.fireFileIngestDone(file.getId());
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
@ -104,8 +101,6 @@ final class FileIngestPipeline {
|
||||
module.shutDown(ingestJobCancelled);
|
||||
} catch (Exception ex) {
|
||||
errors.add(new IngestModuleError(module.getDisplayName(), ex));
|
||||
} finally {
|
||||
IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName());
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
|
@ -46,7 +46,7 @@ final class IngestJob {
|
||||
private ProgressHandle fileTasksProgress;
|
||||
int totalEnqueuedFiles = 0;
|
||||
private int processedFiles = 0;
|
||||
private boolean cancelled;
|
||||
private volatile boolean cancelled;
|
||||
|
||||
IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) {
|
||||
this.id = id;
|
||||
@ -85,7 +85,7 @@ final class IngestJob {
|
||||
"IngestJob.progress.cancelling",
|
||||
displayName));
|
||||
}
|
||||
IngestManager.getInstance().stopAll();
|
||||
IngestManager.getInstance().cancelIngestJobs();
|
||||
return true;
|
||||
}
|
||||
});
|
||||
@ -104,7 +104,7 @@ final class IngestJob {
|
||||
NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
|
||||
displayName));
|
||||
}
|
||||
IngestManager.getInstance().stopAll();
|
||||
IngestManager.getInstance().cancelIngestJobs();
|
||||
return true;
|
||||
}
|
||||
});
|
||||
@ -163,9 +163,9 @@ final class IngestJob {
|
||||
DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId);
|
||||
if (dataSourceIngestPipeline != null) {
|
||||
errors.addAll(dataSourceIngestPipeline.shutDown(cancelled));
|
||||
}
|
||||
dataSourceIngestPipelines.remove(threadId);
|
||||
if (dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) {
|
||||
}
|
||||
if (initialDataSourceIngestPipeline == null && dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) {
|
||||
dataSourceTaskProgress.finish();
|
||||
dataSourceTaskProgress = null;
|
||||
}
|
||||
@ -173,9 +173,9 @@ final class IngestJob {
|
||||
FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId);
|
||||
if (fileIngestPipeline != null) {
|
||||
errors.addAll(fileIngestPipeline.shutDown(cancelled));
|
||||
}
|
||||
fileIngestPipelines.remove(threadId);
|
||||
if (fileIngestPipelines.isEmpty() && fileTasksProgress != null) {
|
||||
}
|
||||
if (initialFileIngestPipeline == null && fileIngestPipelines.isEmpty() && fileTasksProgress != null) {
|
||||
fileTasksProgress.finish();
|
||||
fileTasksProgress = null;
|
||||
}
|
||||
@ -184,14 +184,17 @@ final class IngestJob {
|
||||
}
|
||||
|
||||
synchronized boolean areIngestPipelinesShutDown() {
|
||||
return (dataSourceIngestPipelines.isEmpty() && fileIngestPipelines.isEmpty());
|
||||
return (initialDataSourceIngestPipeline == null
|
||||
&& dataSourceIngestPipelines.isEmpty()
|
||||
&& initialFileIngestPipeline == null
|
||||
&& fileIngestPipelines.isEmpty());
|
||||
}
|
||||
|
||||
synchronized ProgressHandle getDataSourceTaskProgressBar() {
|
||||
return this.dataSourceTaskProgress;
|
||||
}
|
||||
|
||||
synchronized void handleFileTaskStarted(IngestScheduler.FileScheduler.FileTask task) {
|
||||
synchronized void updateFileTasksProgressBar(String currentFileName) {
|
||||
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
|
||||
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
|
||||
totalEnqueuedFiles = newTotalEnqueuedFiles + 1;
|
||||
@ -202,14 +205,23 @@ final class IngestJob {
|
||||
++processedFiles;
|
||||
}
|
||||
|
||||
fileTasksProgress.progress(task.getFile().getName(), processedFiles);
|
||||
fileTasksProgress.progress(currentFileName, processedFiles);
|
||||
}
|
||||
|
||||
synchronized void cancel() {
|
||||
if (initialDataSourceIngestPipeline != null) {
|
||||
initialDataSourceIngestPipeline.shutDown(true);
|
||||
initialDataSourceIngestPipeline = null;
|
||||
}
|
||||
if (initialFileIngestPipeline != null) {
|
||||
initialFileIngestPipeline.shutDown(true);
|
||||
initialFileIngestPipeline = null;
|
||||
}
|
||||
|
||||
cancelled = true;
|
||||
}
|
||||
|
||||
synchronized boolean isCancelled() {
|
||||
boolean isCancelled() {
|
||||
return cancelled;
|
||||
}
|
||||
}
|
||||
|
@ -60,7 +60,7 @@ public final class IngestJobContext {
|
||||
*/
|
||||
public void addFiles(List<AbstractFile> files) {
|
||||
for (AbstractFile file : files) {
|
||||
IngestManager.getInstance().scheduleFile(ingestJob.getId(), file);
|
||||
IngestManager.getInstance().addFileToIngestJob(ingestJob.getId(), file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -232,7 +232,7 @@ public final class IngestJobLauncher {
|
||||
}
|
||||
|
||||
if ((!enabledModuleTemplates.isEmpty()) && (dataSources != null) && (!dataSources.isEmpty())) {
|
||||
IngestManager.getInstance().scheduleDataSourceTasks(dataSources, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace());
|
||||
IngestManager.getInstance().startIngestJobs(dataSources, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -24,10 +24,13 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.logging.Level;
|
||||
import org.openide.util.NbBundle;
|
||||
import org.sleuthkit.autopsy.coreutils.Logger;
|
||||
import javax.swing.SwingWorker;
|
||||
import org.netbeans.api.progress.ProgressHandle;
|
||||
import org.netbeans.api.progress.ProgressHandleFactory;
|
||||
import org.openide.util.Cancellable;
|
||||
@ -36,6 +39,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
|
||||
import org.sleuthkit.datamodel.AbstractFile;
|
||||
import org.sleuthkit.datamodel.Content;
|
||||
import java.util.prefs.Preferences;
|
||||
import javax.swing.SwingWorker;
|
||||
|
||||
/**
|
||||
* Manages the execution of ingest jobs.
|
||||
@ -48,48 +52,115 @@ public class IngestManager {
|
||||
private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2;
|
||||
private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
|
||||
private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class);
|
||||
private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class);
|
||||
private static IngestManager instance;
|
||||
private final IngestScheduler scheduler = IngestScheduler.getInstance();
|
||||
private final IngestMonitor ingestMonitor = new IngestMonitor();
|
||||
private final Preferences userPreferences = NbPreferences.forModule(this.getClass());
|
||||
private final HashMap<Long, IngestJob> ingestJobs = new HashMap<>();
|
||||
private TaskSchedulingWorker taskSchedulingWorker = null;
|
||||
private DataSourceTaskWorker dataSourceTaskWorker = null;
|
||||
private final List<FileTaskWorker> fileTaskWorkers = new ArrayList<>();
|
||||
private long nextDataSourceTaskId = 0;
|
||||
private long nextThreadId = 0;
|
||||
private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor();
|
||||
private final ExecutorService dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor();
|
||||
private final ExecutorService fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS);
|
||||
private final HashMap<Long, IngestJob> ingestJobs = new HashMap<>(); // Maps job ids to jobs
|
||||
private final HashMap<Long, Future<?>> ingestTasks = new HashMap<>(); // Maps task ids to task cancellation handles
|
||||
private AtomicLong ingestJobId = new AtomicLong(0L);
|
||||
private AtomicLong ingestTaskId = new AtomicLong(0L);
|
||||
private volatile IngestUI ingestMessageBox;
|
||||
|
||||
/**
|
||||
* Gets the IngestManager singleton, creating it if necessary.
|
||||
*
|
||||
* @returns The IngestManager singleton.
|
||||
*/
|
||||
public synchronized static IngestManager getInstance() {
|
||||
if (instance == null) {
|
||||
instance = new IngestManager();
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
private IngestManager() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the top component for the ingest messages in box. Called by the
|
||||
* custom installer for this package once the window system is initialized.
|
||||
*/
|
||||
void initIngestMessageInbox() {
|
||||
if (this.ingestMessageBox == null) {
|
||||
this.ingestMessageBox = IngestMessageTopComponent.findInstance();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized static int getNumberOfFileIngestThreads() {
|
||||
return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS);
|
||||
}
|
||||
|
||||
public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) {
|
||||
if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS
|
||||
|| numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) {
|
||||
numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS;
|
||||
}
|
||||
|
||||
userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads);
|
||||
}
|
||||
|
||||
synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
|
||||
if (!isIngestRunning() && ingestMessageBox != null) {
|
||||
ingestMessageBox.clearMessages();
|
||||
}
|
||||
|
||||
long taskId = ingestTaskId.incrementAndGet();
|
||||
Future<?> task = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
|
||||
ingestTasks.put(taskId, task);
|
||||
|
||||
if (ingestMessageBox != null) {
|
||||
ingestMessageBox.restoreMessages();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test if any ingest jobs are in progress.
|
||||
*
|
||||
* @return True if any ingest jobs are in progress, false otherwise
|
||||
*/
|
||||
public boolean isIngestRunning() {
|
||||
return (ingestJobs.isEmpty() == false);
|
||||
}
|
||||
|
||||
synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) {
|
||||
IngestJob job = ingestJobs.get(ingestJobId);
|
||||
if (job != null) {
|
||||
scheduler.getFileScheduler().scheduleFile(job, file);
|
||||
}
|
||||
}
|
||||
|
||||
void cancelIngestJobs() {
|
||||
new IngestCancellationWorker().execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest events.
|
||||
*/
|
||||
public enum IngestEvent {
|
||||
|
||||
// RJCTODO: Update comments
|
||||
/**
|
||||
* Event sent when an ingest module has been started. Second argument of
|
||||
* the property change is a string form of the module name and the third
|
||||
* argument is null.
|
||||
* Property change event fired when an ingest job is started. The ingest
|
||||
* job id is in old value field of the PropertyChangeEvent object.
|
||||
*/
|
||||
STARTED,
|
||||
INGEST_JOB_STARTED,
|
||||
/**
|
||||
* Event sent when an ingest module has completed processing by its own
|
||||
* means. Second argument of the property change is a string form of the
|
||||
* module name and the third argument is null.
|
||||
*
|
||||
* This event is generally used by listeners to perform a final data
|
||||
* view refresh (listeners need to query all data from the blackboard).
|
||||
* Property change event fired when an ingest job is completed. The
|
||||
* ingest job id is in old value field of the PropertyChangeEvent
|
||||
* object.
|
||||
*/
|
||||
COMPLETED,
|
||||
INGEST_JOB_COMPLETED,
|
||||
/**
|
||||
* Event sent when an ingest module has stopped processing, and likely
|
||||
* not all data has been processed. Second argument of the property
|
||||
* change is a string form of the module name and third argument is
|
||||
* null.
|
||||
* Property change event fired when an ingest job is canceled. The
|
||||
* ingest job id is in old value field of the PropertyChangeEvent
|
||||
* object.
|
||||
*/
|
||||
STOPPED,
|
||||
INGEST_JOB_CANCELLED,
|
||||
/**
|
||||
* Event sent when ingest module posts new data to blackboard or
|
||||
* Event sent when an ingest module posts new data to blackboard or
|
||||
* somewhere else. Second argument of the property change fired contains
|
||||
* ModuleDataEvent object and third argument is null. The object can
|
||||
* contain encapsulated new data created by the module. Listener can
|
||||
@ -109,67 +180,22 @@ public class IngestManager {
|
||||
FILE_DONE,
|
||||
};
|
||||
|
||||
private IngestManager() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns reference to singleton instance.
|
||||
*
|
||||
* @returns Instance of class.
|
||||
*/
|
||||
synchronized public static IngestManager getInstance() {
|
||||
if (instance == null) {
|
||||
instance = new IngestManager();
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* called by Installer in AWT thread once the Window System is ready
|
||||
*/
|
||||
void initIngestMessageInbox() {
|
||||
if (this.ingestMessageBox == null) {
|
||||
this.ingestMessageBox = IngestMessageTopComponent.findInstance();
|
||||
}
|
||||
}
|
||||
|
||||
synchronized private long getNextDataSourceTaskId() {
|
||||
return ++this.nextDataSourceTaskId;
|
||||
}
|
||||
|
||||
synchronized private long getNextThreadId() {
|
||||
return ++this.nextThreadId;
|
||||
}
|
||||
|
||||
public synchronized int getNumberOfFileIngestThreads() {
|
||||
return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS);
|
||||
}
|
||||
|
||||
public synchronized void setNumberOfFileIngestThreads(int numberOfThreads) {
|
||||
if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS
|
||||
|| numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) {
|
||||
numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS;
|
||||
}
|
||||
userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add property change listener to listen to ingest events as defined in
|
||||
* IngestModuleEvent.
|
||||
* Add property change listener to listen to ingest events.
|
||||
*
|
||||
* @param listener PropertyChangeListener to register
|
||||
*/
|
||||
public static synchronized void addPropertyChangeListener(final PropertyChangeListener listener) {
|
||||
public static void addPropertyChangeListener(final PropertyChangeListener listener) {
|
||||
pcs.addPropertyChangeListener(listener);
|
||||
}
|
||||
|
||||
public static synchronized void removePropertyChangeListener(final PropertyChangeListener listener) {
|
||||
public static void removePropertyChangeListener(final PropertyChangeListener listener) {
|
||||
pcs.removePropertyChangeListener(listener);
|
||||
}
|
||||
|
||||
static synchronized void fireModuleEvent(String eventType, String moduleName) {
|
||||
static void fireIngestJobEvent(String eventType, long jobId) {
|
||||
try {
|
||||
pcs.firePropertyChange(eventType, moduleName, null);
|
||||
pcs.firePropertyChange(eventType, jobId, null);
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e);
|
||||
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
|
||||
@ -181,11 +207,11 @@ public class IngestManager {
|
||||
/**
|
||||
* Fire event when file is done with a pipeline run
|
||||
*
|
||||
* @param objId ID of file that is done
|
||||
* @param fileId ID of file that is done
|
||||
*/
|
||||
static synchronized void fireFileDone(long objId) {
|
||||
static void fireFileIngestDone(long fileId) {
|
||||
try {
|
||||
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), objId, null);
|
||||
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null);
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e);
|
||||
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
|
||||
@ -200,7 +226,7 @@ public class IngestManager {
|
||||
*
|
||||
* @param moduleDataEvent
|
||||
*/
|
||||
static synchronized void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
|
||||
static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
|
||||
try {
|
||||
pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null);
|
||||
} catch (Exception e) {
|
||||
@ -217,7 +243,7 @@ public class IngestManager {
|
||||
*
|
||||
* @param moduleContentEvent
|
||||
*/
|
||||
static synchronized void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
|
||||
static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
|
||||
try {
|
||||
pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null);
|
||||
} catch (Exception e) {
|
||||
@ -228,218 +254,6 @@ public class IngestManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Multiple data-sources version of scheduleDataSource() method. Enqueues
|
||||
* multiple sources inputs (Content objects) and associated modules at once
|
||||
*
|
||||
* @param modules modules to scheduleDataSource on every data source
|
||||
* @param inputs input data sources to enqueue and scheduleDataSource the
|
||||
* ingest modules on
|
||||
*/
|
||||
void scheduleDataSourceTasks(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
|
||||
if (!isIngestRunning() && ingestMessageBox != null) {
|
||||
ingestMessageBox.clearMessages();
|
||||
}
|
||||
|
||||
taskSchedulingWorker = new TaskSchedulingWorker(dataSources, moduleTemplates, processUnallocatedSpace);
|
||||
taskSchedulingWorker.execute();
|
||||
|
||||
if (ingestMessageBox != null) {
|
||||
ingestMessageBox.restoreMessages();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* IngestManager entry point, enqueues data to be processed and starts new
|
||||
* ingest as needed, or just enqueues data to an existing pipeline.
|
||||
*
|
||||
* Spawns background thread which enumerates all sorted files and executes
|
||||
* chosen modules per file in a pre-determined order. Notifies modules when
|
||||
* work is complete or should be interrupted using complete() and stop()
|
||||
* calls. Does not block and can be called multiple times to enqueue more
|
||||
* work to already running background ingest process.
|
||||
*
|
||||
* @param modules modules to scheduleDataSource on the data source input
|
||||
* @param input input data source Content objects to scheduleDataSource the
|
||||
* ingest modules on
|
||||
*/
|
||||
void scheduleDataSourceTask(final Content dataSource, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
|
||||
List<Content> dataSources = new ArrayList<>();
|
||||
dataSources.add(dataSource);
|
||||
scheduleDataSourceTasks(dataSources, moduleTemplates, processUnallocatedSpace);
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule a file for ingest and add it to ongoing file ingest process on
|
||||
* the same data source. Scheduler updates the current progress.
|
||||
*
|
||||
* The file to be added is usually a product of a currently ran ingest. Now
|
||||
* we want to process this new file with the same ingest context.
|
||||
*
|
||||
* @param file file to be scheduled
|
||||
* @param pipelineContext ingest context used to ingest parent of the file
|
||||
* to be scheduled
|
||||
*/
|
||||
void scheduleFile(long ingestJobId, AbstractFile file) {
|
||||
IngestJob job = this.ingestJobs.get(ingestJobId);
|
||||
if (job == null) {
|
||||
logger.log(Level.SEVERE, "Unable to map ingest job id (id = {0}) to an ingest job, failed to schedule file (id = {1})", new Object[]{ingestJobId, file.getId()});
|
||||
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
|
||||
"Unable to associate " + file.getName() + " with ingest job, file will not be processed by ingest nodules",
|
||||
MessageNotifyUtil.MessageType.ERROR);
|
||||
}
|
||||
|
||||
scheduler.getFileScheduler().scheduleFile(job, file);
|
||||
}
|
||||
|
||||
private synchronized void startAll() {
|
||||
// Make sure the ingest monitor is running.
|
||||
if (!ingestMonitor.isRunning()) {
|
||||
ingestMonitor.start();
|
||||
}
|
||||
|
||||
// Make sure a data source task worker is running.
|
||||
// TODO: There is a race condition here with SwingWorker.isDone().
|
||||
// The highly unlikely chance that no data source task worker will
|
||||
// run for this job needs to be addressed. Fix by using a thread pool
|
||||
// and converting the SwingWorkers to Runnables.
|
||||
if (dataSourceTaskWorker == null || dataSourceTaskWorker.isDone()) {
|
||||
dataSourceTaskWorker = new DataSourceTaskWorker(getNextThreadId());
|
||||
dataSourceTaskWorker.execute();
|
||||
}
|
||||
|
||||
// Make sure the requested number of file task workers are running.
|
||||
// TODO: There is a race condition here with SwingWorker.isDone().
|
||||
// The highly unlikely chance that no file task workers or the wrong
|
||||
// number of file task workers will run for this job needs to be
|
||||
// addressed. Fix by using a thread pool and converting the SwingWorkers
|
||||
// to Runnables.
|
||||
int workersRequested = getNumberOfFileIngestThreads();
|
||||
int workersRunning = 0;
|
||||
for (FileTaskWorker worker : fileTaskWorkers) {
|
||||
if (worker != null) {
|
||||
if (worker.isDone()) {
|
||||
if (workersRunning < workersRequested) {
|
||||
worker = new FileTaskWorker(getNextThreadId());
|
||||
worker.execute();
|
||||
++workersRunning;
|
||||
} else {
|
||||
worker = null;
|
||||
}
|
||||
} else {
|
||||
++workersRunning;
|
||||
}
|
||||
} else if (workersRunning < workersRequested) {
|
||||
worker = new FileTaskWorker(getNextThreadId());
|
||||
worker.execute();
|
||||
++workersRunning;
|
||||
}
|
||||
}
|
||||
while (workersRunning < workersRequested
|
||||
&& fileTaskWorkers.size() < MAX_NUMBER_OF_FILE_INGEST_THREADS) {
|
||||
FileTaskWorker worker = new FileTaskWorker(getNextThreadId());
|
||||
fileTaskWorkers.add(worker);
|
||||
worker.execute();
|
||||
++workersRunning;
|
||||
}
|
||||
}
|
||||
|
||||
synchronized void reportThreadDone(long threadId) {
|
||||
List<Long> completedJobs = new ArrayList<>();
|
||||
for (IngestJob job : ingestJobs.values()) {
|
||||
job.releaseIngestPipelinesForThread(threadId);
|
||||
if (job.areIngestPipelinesShutDown()) {
|
||||
completedJobs.add(job.getId());
|
||||
}
|
||||
}
|
||||
|
||||
for (Long jobId : completedJobs) {
|
||||
ingestJobs.remove(jobId);
|
||||
}
|
||||
}
|
||||
|
||||
synchronized void stopAll() {
|
||||
// First get the task scheduling worker to stop adding new tasks.
|
||||
if (taskSchedulingWorker != null) {
|
||||
taskSchedulingWorker.cancel(true);
|
||||
while (!taskSchedulingWorker.isDone()) {
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException ex) {
|
||||
}
|
||||
}
|
||||
taskSchedulingWorker = null;
|
||||
}
|
||||
|
||||
// Now mark all of the ingest jobs as cancelled. This way the ingest
|
||||
// modules will know they are being shut down due to cancellation when
|
||||
// the cancelled ingest workers release their pipelines.
|
||||
for (IngestJob job : ingestJobs.values()) {
|
||||
job.cancel();
|
||||
}
|
||||
|
||||
// Cancel the data source task worker. It will release its pipelines
|
||||
// in its done() method and the pipelines will shut down their modules.
|
||||
if (dataSourceTaskWorker != null) {
|
||||
dataSourceTaskWorker.cancel(true);
|
||||
while (!dataSourceTaskWorker.isDone()) {
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException ex) {
|
||||
}
|
||||
}
|
||||
dataSourceTaskWorker = null;
|
||||
}
|
||||
|
||||
// Cancel the file task workers. They will release their pipelines
|
||||
// in their done() methods and the pipelines will shut down their
|
||||
// modules.
|
||||
for (FileTaskWorker worker : fileTaskWorkers) {
|
||||
if (worker != null) {
|
||||
worker.cancel(true);
|
||||
while (!worker.isDone()) {
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException ex) {
|
||||
}
|
||||
}
|
||||
worker = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Jettision the remaining tasks. This will dispose of any tasks that
|
||||
// the scheduling worker queued up before it was cancelled.
|
||||
scheduler.getFileScheduler().empty();
|
||||
scheduler.getDataSourceScheduler().empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Test if any ingest modules are running
|
||||
*
|
||||
* @return true if any module is running, false otherwise
|
||||
*/
|
||||
public synchronized boolean isIngestRunning() {
|
||||
// TODO: There is a race condition here with SwingWorker.isDone().
|
||||
// It probably needs to be addressed at a later date. If we replace the
|
||||
// SwingWorkers with a thread pool and Runnables, one solution would be
|
||||
// to check the ingest jobs list.
|
||||
if (taskSchedulingWorker != null && !taskSchedulingWorker.isDone()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (dataSourceTaskWorker != null && !dataSourceTaskWorker.isDone()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
for (FileTaskWorker worker : fileTaskWorkers) {
|
||||
if (worker != null && !worker.isDone()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Module publishes message using InegestManager handle Does not block. The
|
||||
* message gets enqueued in the GUI thread and displayed in a widget
|
||||
@ -468,47 +282,102 @@ public class IngestManager {
|
||||
}
|
||||
}
|
||||
|
||||
private class TaskSchedulingWorker extends SwingWorker<Object, Void> {
|
||||
private synchronized void startIngestTasks() {
|
||||
if (!ingestMonitor.isRunning()) {
|
||||
ingestMonitor.start();
|
||||
}
|
||||
|
||||
long taskId = ingestTaskId.incrementAndGet();
|
||||
Future<?> task = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId));
|
||||
ingestTasks.put(taskId, task);
|
||||
|
||||
int numberOfFileTasksRequested = getNumberOfFileIngestThreads();
|
||||
for (int i = 0; i < numberOfFileTasksRequested; ++i) {
|
||||
taskId = ingestTaskId.incrementAndGet();
|
||||
task = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId));
|
||||
ingestTasks.put(taskId, task);
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void stopIngestTasks() {
|
||||
// First mark all of the ingest jobs as cancelled. This way the
|
||||
// ingest modules will know they are being shut down due to
|
||||
// cancellation when the cancelled run ingest module tasks release
|
||||
// their pipelines.
|
||||
for (IngestJob job : ingestJobs.values()) {
|
||||
job.cancel();
|
||||
}
|
||||
|
||||
// Cancel the run ingest module tasks, setting the state of the threads
|
||||
// running them to interrupted.
|
||||
for (Future<?> task : ingestTasks.values()) {
|
||||
task.cancel(true);
|
||||
}
|
||||
|
||||
// Jettision the remaining data source and file ingest tasks.
|
||||
scheduler.getFileScheduler().empty();
|
||||
scheduler.getDataSourceScheduler().empty();
|
||||
}
|
||||
|
||||
synchronized void reportStartIngestJobsTaskDone(long taskId) {
|
||||
ingestTasks.remove(taskId);
|
||||
}
|
||||
|
||||
synchronized void reportRunIngestModulesTaskDone(long taskId) {
|
||||
ingestTasks.remove(taskId);
|
||||
|
||||
List<Long> completedJobs = new ArrayList<>();
|
||||
for (IngestJob job : ingestJobs.values()) {
|
||||
job.releaseIngestPipelinesForThread(taskId);
|
||||
if (job.areIngestPipelinesShutDown() == true) {
|
||||
completedJobs.add(job.getId());
|
||||
}
|
||||
}
|
||||
|
||||
for (Long jobId : completedJobs) {
|
||||
IngestJob job = ingestJobs.remove(jobId);
|
||||
fireIngestJobEvent(job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED.toString() : IngestEvent.INGEST_JOB_COMPLETED.toString(), jobId);
|
||||
}
|
||||
}
|
||||
|
||||
private class StartIngestJobsTask implements Runnable {
|
||||
|
||||
private final long id;
|
||||
private final List<Content> dataSources;
|
||||
private final List<IngestModuleTemplate> moduleTemplates;
|
||||
private final boolean processUnallocatedSpace;
|
||||
private ProgressHandle progress;
|
||||
|
||||
TaskSchedulingWorker(List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
|
||||
StartIngestJobsTask(long taskId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
|
||||
this.id = taskId;
|
||||
this.dataSources = dataSources;
|
||||
this.moduleTemplates = moduleTemplates;
|
||||
this.processUnallocatedSpace = processUnallocatedSpace;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object doInBackground() throws Exception {
|
||||
// Set up a progress bar that can be used to cancel all of the
|
||||
// ingest jobs currently being performed.
|
||||
public void run() {
|
||||
try {
|
||||
final String displayName = "Queueing ingest tasks";
|
||||
progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
|
||||
@Override
|
||||
public boolean cancel() {
|
||||
logger.log(Level.INFO, "Queueing ingest cancelled by user.");
|
||||
if (progress != null) {
|
||||
progress.setDisplayName(displayName + " (Cancelling...)");
|
||||
}
|
||||
IngestManager.getInstance().stopAll();
|
||||
IngestManager.getInstance().cancelIngestJobs();
|
||||
return true;
|
||||
}
|
||||
});
|
||||
|
||||
progress.start(2 * dataSources.size());
|
||||
int processed = 0;
|
||||
int workUnitsCompleted = 0;
|
||||
for (Content dataSource : dataSources) {
|
||||
if (isCancelled()) {
|
||||
logger.log(Level.INFO, "Task scheduling thread cancelled");
|
||||
return null;
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
break;
|
||||
}
|
||||
|
||||
final String inputName = dataSource.getName();
|
||||
IngestJob ingestJob = new IngestJob(IngestManager.this.getNextDataSourceTaskId(), dataSource, moduleTemplates, processUnallocatedSpace);
|
||||
|
||||
IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace);
|
||||
List<IngestModuleError> errors = ingestJob.startUpIngestPipelines();
|
||||
if (!errors.isEmpty()) {
|
||||
StringBuilder failedModules = new StringBuilder();
|
||||
@ -521,29 +390,111 @@ public class IngestManager {
|
||||
failedModules.append(",");
|
||||
}
|
||||
}
|
||||
MessageNotifyUtil.Message.error(
|
||||
MessageNotifyUtil.Message.error( // RJCTODO: Fix this to show all errors, probably should specify data source name
|
||||
"Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n"
|
||||
+ "No ingest modules will be run. Please disable the module "
|
||||
+ "or fix the error and restart ingest by right clicking on "
|
||||
+ "the data source and selecting Run Ingest Modules.\n\n"
|
||||
+ "Error: " + errors.get(0).getModuleError().getMessage());
|
||||
return null;
|
||||
ingestJob.cancel();
|
||||
break;
|
||||
}
|
||||
|
||||
// Save the ingest job for later cleanup of pipelines.
|
||||
synchronized (IngestManager.this) {
|
||||
ingestJobs.put(ingestJob.getId(), ingestJob);
|
||||
|
||||
// Queue the data source ingest tasks for the ingest job.
|
||||
progress.progress("DataSource Ingest" + " " + inputName, processed);
|
||||
scheduler.getDataSourceScheduler().schedule(ingestJob);
|
||||
progress.progress("DataSource Ingest" + " " + inputName, ++processed);
|
||||
|
||||
// Queue the file ingest tasks for the ingest job.
|
||||
progress.progress("File Ingest" + " " + inputName, processed);
|
||||
scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob);
|
||||
progress.progress("File Ingest" + " " + inputName, ++processed);
|
||||
}
|
||||
|
||||
// Queue the data source ingest tasks for the ingest job.
|
||||
final String inputName = dataSource.getName();
|
||||
progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); // RJCTODO: Improve
|
||||
scheduler.getDataSourceScheduler().schedule(ingestJob);
|
||||
progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted);
|
||||
|
||||
// Queue the file ingest tasks for the ingest job.
|
||||
progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted);
|
||||
scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob);
|
||||
progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted);
|
||||
|
||||
if (!Thread.currentThread().isInterrupted()) {
|
||||
startIngestTasks();
|
||||
fireIngestJobEvent(IngestEvent.INGEST_JOB_STARTED.toString(), ingestJob.getId());
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
String message = String.format("StartIngestJobsTask (id=%d) caught exception", id);
|
||||
logger.log(Level.SEVERE, message, ex);
|
||||
MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial");
|
||||
} finally {
|
||||
progress.finish();
|
||||
reportStartIngestJobsTaskDone(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class RunDataSourceIngestModulesTask implements Runnable {
|
||||
|
||||
private final long id;
|
||||
|
||||
RunDataSourceIngestModulesTask(long taskId) {
|
||||
id = taskId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler();
|
||||
while (scheduler.hasNext()) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
break;
|
||||
}
|
||||
IngestJob job = scheduler.next();
|
||||
job.getDataSourceIngestPipelineForThread(id).process();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id);
|
||||
logger.log(Level.SEVERE, message, ex);
|
||||
} finally {
|
||||
reportRunIngestModulesTaskDone(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class RunFileSourceIngestModulesTask implements Runnable {
|
||||
|
||||
private final long id;
|
||||
|
||||
RunFileSourceIngestModulesTask(long taskId) {
|
||||
id = taskId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler();
|
||||
while (fileScheduler.hasNext()) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
break;
|
||||
}
|
||||
IngestScheduler.FileScheduler.FileTask task = fileScheduler.next();
|
||||
IngestJob job = task.getJob();
|
||||
job.updateFileTasksProgressBar(task.getFile().getName());
|
||||
job.getFileIngestPipelineForThread(id).process(task.getFile());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id);
|
||||
logger.log(Level.SEVERE, message, ex);
|
||||
} finally {
|
||||
reportRunIngestModulesTaskDone(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class IngestCancellationWorker extends SwingWorker<Void, Void> {
|
||||
|
||||
@Override
|
||||
protected Void doInBackground() throws Exception {
|
||||
stopIngestTasks();
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -552,105 +503,8 @@ public class IngestManager {
|
||||
try {
|
||||
super.get();
|
||||
} catch (CancellationException | InterruptedException ex) {
|
||||
// IngestManager.stopAll() will dispose of all tasks.
|
||||
} catch (Exception ex) {
|
||||
logger.log(Level.SEVERE, "Error while scheduling ingest jobs", ex);
|
||||
MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial");
|
||||
} finally {
|
||||
if (!isCancelled()) {
|
||||
startAll();
|
||||
}
|
||||
progress.finish();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs data source ingest tasks for one or more ingest jobs on a worker
|
||||
* thread.
|
||||
*/
|
||||
class DataSourceTaskWorker extends SwingWorker<Object, Void> {
|
||||
|
||||
private final long id;
|
||||
|
||||
DataSourceTaskWorker(long threadId) {
|
||||
this.id = threadId;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Void doInBackground() throws Exception {
|
||||
logger.log(Level.INFO, "Data source ingest thread (id={0}) started", this.id);
|
||||
IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler();
|
||||
while (scheduler.hasNext()) {
|
||||
if (isCancelled()) {
|
||||
logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id);
|
||||
return null;
|
||||
}
|
||||
IngestJob job = scheduler.next();
|
||||
DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(this.id);
|
||||
pipeline.process(this, job.getDataSourceTaskProgressBar());
|
||||
}
|
||||
logger.log(Level.INFO, "Data source ingest thread (id={0}) completed", this.id);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void done() {
|
||||
try {
|
||||
super.get();
|
||||
} catch (CancellationException | InterruptedException e) {
|
||||
logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id);
|
||||
} catch (Exception ex) {
|
||||
String message = String.format("Data source ingest thread (id=%d) experienced a fatal error", this.id);
|
||||
logger.log(Level.SEVERE, message, ex);
|
||||
} finally {
|
||||
IngestManager.getInstance().reportThreadDone(this.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs file ingest tasks for one or more ingest jobs on a worker
|
||||
* thread.
|
||||
*/
|
||||
class FileTaskWorker extends SwingWorker<Object, Void> {
|
||||
|
||||
private final long id;
|
||||
|
||||
FileTaskWorker(long threadId) {
|
||||
this.id = threadId;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object doInBackground() throws Exception {
|
||||
logger.log(Level.INFO, "File ingest thread (id={0}) started", this.id);
|
||||
IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler();
|
||||
while (fileScheduler.hasNext()) {
|
||||
if (isCancelled()) {
|
||||
logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id);
|
||||
return null;
|
||||
}
|
||||
IngestScheduler.FileScheduler.FileTask task = fileScheduler.next();
|
||||
IngestJob job = task.getJob();
|
||||
FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(this.id);
|
||||
job.handleFileTaskStarted(task);
|
||||
pipeline.process(task.getFile());
|
||||
}
|
||||
logger.log(Level.INFO, "File ingest thread (id={0}) completed", this.id);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void done() {
|
||||
try {
|
||||
super.get();
|
||||
} catch (CancellationException | InterruptedException e) {
|
||||
logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id);
|
||||
} catch (Exception ex) {
|
||||
String message = String.format("File ingest thread {0} experienced a fatal error", this.id);
|
||||
logger.log(Level.SEVERE, message, ex);
|
||||
} finally {
|
||||
IngestManager.getInstance().reportThreadDone(this.id);
|
||||
logger.log(Level.SEVERE, "Error while cancelling ingest jobs", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content;
|
||||
manager = IngestManager.getInstance();
|
||||
}
|
||||
try {
|
||||
manager.stopAll();
|
||||
manager.cancelIngestJobs();
|
||||
} finally {
|
||||
//clear inbox
|
||||
clearMessages();
|
||||
|
@ -163,9 +163,9 @@ public final class IngestMonitor {
|
||||
if (checkDiskSpace() == false) {
|
||||
//stop ingest if running
|
||||
final String diskPath = root.getAbsolutePath();
|
||||
MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath);
|
||||
logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath);
|
||||
manager.stopAll();
|
||||
MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath);
|
||||
logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath);
|
||||
manager.cancelIngestJobs();
|
||||
IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage(
|
||||
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath),
|
||||
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath)));
|
||||
|
@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.hashdatabase;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.Component;
|
||||
import java.awt.EventQueue;
|
||||
import java.awt.Frame;
|
||||
import java.awt.event.KeyEvent;
|
||||
import java.beans.PropertyChangeEvent;
|
||||
@ -71,9 +72,14 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
|
||||
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
|
||||
@Override
|
||||
public void propertyChange(PropertyChangeEvent evt) {
|
||||
if (isFileIngestStatusChangeEvent(evt)) {
|
||||
if (isIngestJobEvent(evt)) {
|
||||
EventQueue.invokeLater(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
updateComponents();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@ -224,8 +230,10 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
|
||||
return shortenedPath;
|
||||
}
|
||||
|
||||
private boolean isFileIngestStatusChangeEvent(PropertyChangeEvent evt) {
|
||||
return evt.getPropertyName().equals(IngestManager.IngestEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.COMPLETED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.STOPPED.toString());
|
||||
private boolean isIngestJobEvent(PropertyChangeEvent evt) {
|
||||
return evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_STARTED.toString())
|
||||
|| evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_COMPLETED.toString())
|
||||
|| evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_CANCELLED.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -18,6 +18,7 @@
|
||||
*/
|
||||
package org.sleuthkit.autopsy.keywordsearch;
|
||||
|
||||
import java.awt.EventQueue;
|
||||
import java.awt.event.ActionEvent;
|
||||
import java.awt.event.ActionListener;
|
||||
import java.beans.PropertyChangeEvent;
|
||||
@ -52,7 +53,6 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
|
||||
private static Logger logger = Logger.getLogger(KeywordSearchEditListPanel.class.getName());
|
||||
private KeywordTableModel tableModel;
|
||||
private KeywordList currentKeywordList;
|
||||
private boolean ingestRunning;
|
||||
|
||||
/**
|
||||
* Creates new form KeywordSearchEditListPanel
|
||||
@ -101,7 +101,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
|
||||
}
|
||||
});
|
||||
|
||||
initButtons();
|
||||
setButtonStates();
|
||||
|
||||
addWordField.setComponentPopupMenu(rightClickMenu);
|
||||
ActionListener actList = new ActionListener() {
|
||||
@ -124,49 +124,28 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
|
||||
pasteMenuItem.addActionListener(actList);
|
||||
selectAllMenuItem.addActionListener(actList);
|
||||
|
||||
if (IngestManager.getInstance().isIngestRunning()) {
|
||||
initIngest(0);
|
||||
} else {
|
||||
initIngest(1);
|
||||
}
|
||||
setButtonStates();
|
||||
|
||||
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
|
||||
@Override
|
||||
public void propertyChange(PropertyChangeEvent evt) {
|
||||
String changed = evt.getPropertyName();
|
||||
Object oldValue = evt.getOldValue();
|
||||
if (changed.equals(IngestEvent.COMPLETED.toString())
|
||||
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
|
||||
initIngest(1);
|
||||
} else if (changed.equals(IngestEvent.STARTED.toString())
|
||||
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
|
||||
initIngest(0);
|
||||
} else if (changed.equals(IngestEvent.STOPPED.toString())
|
||||
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
|
||||
initIngest(1);
|
||||
if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString())
|
||||
|| changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
|
||||
|| changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
|
||||
EventQueue.invokeLater(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
setButtonStates();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize this panel depending on whether ingest is running
|
||||
*
|
||||
* @param running case 0: ingest running case 1: ingest not running
|
||||
*/
|
||||
private void initIngest(int running) {
|
||||
switch (running) {
|
||||
case 0:
|
||||
ingestRunning = true;
|
||||
break;
|
||||
case 1:
|
||||
ingestRunning = false;
|
||||
break;
|
||||
}
|
||||
initButtons();
|
||||
}
|
||||
|
||||
void initButtons() {
|
||||
void setButtonStates() {
|
||||
boolean ingestRunning = IngestManager.getInstance().isIngestRunning();
|
||||
boolean listSet = currentKeywordList != null;
|
||||
boolean isLocked = !listSet ? true : currentKeywordList.isLocked();
|
||||
boolean noKeywords = !listSet ? true : currentKeywordList.getKeywords().isEmpty();
|
||||
@ -441,7 +420,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
|
||||
chRegex.setSelected(false);
|
||||
addWordField.setText("");
|
||||
|
||||
initButtons();
|
||||
setButtonStates();
|
||||
}//GEN-LAST:event_addWordButtonActionPerformed
|
||||
|
||||
private void deleteWordButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteWordButtonActionPerformed
|
||||
@ -449,7 +428,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
|
||||
|
||||
tableModel.deleteSelected(keywordTable.getSelectedRows());
|
||||
KeywordSearchListsXML.getCurrent().addList(currentKeywordList);
|
||||
initButtons();
|
||||
setButtonStates();
|
||||
}
|
||||
}//GEN-LAST:event_deleteWordButtonActionPerformed
|
||||
|
||||
@ -549,11 +528,11 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
|
||||
|
||||
currentKeywordList = loader.getListsL(false).get(index);
|
||||
tableModel.resync();
|
||||
initButtons();
|
||||
setButtonStates();
|
||||
} else {
|
||||
currentKeywordList = null;
|
||||
tableModel.resync();
|
||||
initButtons();
|
||||
setButtonStates();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -44,7 +44,7 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl
|
||||
if (KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.title"), NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.body"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) {
|
||||
String toDelete = editListPanel.getCurrentKeywordList().getName();
|
||||
editListPanel.setCurrentKeywordList(null);
|
||||
editListPanel.initButtons();
|
||||
editListPanel.setButtonStates();
|
||||
// RJCTODO: Move this into a deleteList method in the manager
|
||||
KeywordSearchListsXML deleter = KeywordSearchListsXML.getCurrent();
|
||||
deleter.deleteList(toDelete);
|
||||
|
@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearch;
|
||||
|
||||
import java.awt.Component;
|
||||
import java.awt.Cursor;
|
||||
import java.awt.EventQueue;
|
||||
import java.awt.event.ActionEvent;
|
||||
import java.awt.event.ActionListener;
|
||||
import java.beans.PropertyChangeEvent;
|
||||
@ -114,26 +115,23 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
|
||||
}
|
||||
});
|
||||
|
||||
if (IngestManager.getInstance().isIngestRunning()) {
|
||||
initIngest(true);
|
||||
} else {
|
||||
initIngest(false);
|
||||
}
|
||||
ingestRunning = IngestManager.getInstance().isIngestRunning();
|
||||
updateComponents();
|
||||
|
||||
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
|
||||
@Override
|
||||
public void propertyChange(PropertyChangeEvent evt) {
|
||||
String changed = evt.getPropertyName();
|
||||
Object oldValue = evt.getOldValue();
|
||||
if (changed.equals(IngestEvent.COMPLETED.toString())
|
||||
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
|
||||
initIngest(false);
|
||||
} else if (changed.equals(IngestEvent.STARTED.toString())
|
||||
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
|
||||
initIngest(true);
|
||||
} else if (changed.equals(IngestEvent.STOPPED.toString())
|
||||
&& ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
|
||||
initIngest(false);
|
||||
if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString())
|
||||
|| changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
|
||||
|| changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
|
||||
EventQueue.invokeLater(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
ingestRunning = IngestManager.getInstance().isIngestRunning();
|
||||
updateComponents();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
@ -152,28 +150,21 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
|
||||
searchAddButton.addActionListener(searchAddListener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize this panel depending on whether ingest is running
|
||||
*
|
||||
* @param running case 0: ingest running case 1: ingest not running
|
||||
*/
|
||||
private void initIngest(boolean running) {
|
||||
if (running) {
|
||||
ingestRunning = true;
|
||||
private void updateComponents() {
|
||||
ingestRunning = IngestManager.getInstance().isIngestRunning();
|
||||
if (ingestRunning) {
|
||||
searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestTitle"));
|
||||
searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestMsg" ));
|
||||
listsTableModel.resync();
|
||||
|
||||
} else {
|
||||
ingestRunning = false;
|
||||
searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.searchIngestTitle"));
|
||||
searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIdxSearchMsg"));
|
||||
listsTableModel.resync();
|
||||
}
|
||||
updateIngestIndexLabel(running);
|
||||
listsTableModel.resync();
|
||||
updateIngestIndexLabel();
|
||||
}
|
||||
|
||||
private void updateIngestIndexLabel(boolean ingestRunning) {
|
||||
private void updateIngestIndexLabel() {
|
||||
if (ingestRunning) {
|
||||
ingestIndexLabel.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.ongoingIngestMsg", filesIndexed));
|
||||
}
|
||||
@ -184,7 +175,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
|
||||
|
||||
@Override
|
||||
protected void postFilesIndexedChange() {
|
||||
updateIngestIndexLabel(ingestRunning);
|
||||
updateIngestIndexLabel();
|
||||
}
|
||||
|
||||
/**
|
||||
|
Loading…
x
Reference in New Issue
Block a user