mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-12 16:06:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4700 from rcordovano/4932-ain-input-scan-performance-fix

Browse Source 
Restore less comprehensive manifest file locking behavior in AIM
This commit is contained in:
Richard Cordovano 2019-04-10 10:27:48 -04:00 committed by GitHub
commit 803da45d6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 202 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

376
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

@ -127,7 +127,6 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
private static final int NUM_INPUT_SCAN_SCHEDULING_THREADS = 1; private static final int NUM_INPUT_SCAN_SCHEDULING_THREADS = 1;
private static final String INPUT_SCAN_SCHEDULER_THREAD_NAME = "AIM-input-scan-scheduler-%d"; private static final String INPUT_SCAN_SCHEDULER_THREAD_NAME = "AIM-input-scan-scheduler-%d";
private static final String INPUT_SCAN_THREAD_NAME = "AIM-input-scan-%d"; private static final String INPUT_SCAN_THREAD_NAME = "AIM-input-scan-%d";
private static final int INPUT_SCAN_LOCKING_TIMEOUT_MINS = 5;
private static final String AUTO_INGEST_THREAD_NAME = "AIM-job-processing-%d"; private static final String AUTO_INGEST_THREAD_NAME = "AIM-job-processing-%d";
private static final String LOCAL_HOST_NAME = NetworkUtils.getLocalHostName(); private static final String LOCAL_HOST_NAME = NetworkUtils.getLocalHostName();
private static final String EVENT_CHANNEL_NAME = "Auto-Ingest-Manager-Events"; private static final String EVENT_CHANNEL_NAME = "Auto-Ingest-Manager-Events";
@ -175,6 +174,12 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
private volatile AutoIngestNodeStateEvent lastPublishedStateEvent; private volatile AutoIngestNodeStateEvent lastPublishedStateEvent;
/**
* Gets the name of the file in a case directory that is used to record the
* manifest file paths for the auto ingest jobs for the case.
*
* @return The file name.
*/
static String getCaseManifestsListFileName() { static String getCaseManifestsListFileName() {
return CASE_MANIFESTS_LIST_FILE_NAME; return CASE_MANIFESTS_LIST_FILE_NAME;
} }
@ -1013,15 +1018,16 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
} }
/** /**
* Sets the error flag for case node data given a case directory path. * Sets the error flag in the case node data stored in a case directory
* coordination service node.
* *
* @param caseDirectoryPath The case directory path. * @param caseDirectoryPath The case directory path.
* *
* @throws InterruptedException If the thread running the input directory * @throws InterruptedException If the thread running the input directory
* scan task is interrupted while blocked, * scan task is interrupted while blocked,
* i.e., if auto ingest is shutting down. * i.e., if auto ingest is shutting down.
*/ */
private void setCaseNodeDataErrorsOccurred(Path caseDirectoryPath) throws InterruptedException { private void setErrorsOccurredFlagForCase(Path caseDirectoryPath) throws InterruptedException {
try { try {
CaseNodeData caseNodeData = CaseNodeData.readCaseNodeData(caseDirectoryPath.toString()); CaseNodeData caseNodeData = CaseNodeData.readCaseNodeData(caseDirectoryPath.toString());
caseNodeData.setErrorsOccurred(true); caseNodeData.setErrorsOccurred(true);
@ -1158,8 +1164,8 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
* @param filePath The path of the file. * @param filePath The path of the file.
* @param attrs The file system attributes of the file. * @param attrs The file system attributes of the file.
* *
* @return TERMINATE if auto ingest is shutting down, CONTINUE if it has * @return TERMINATE if auto ingest is shutting down, CONTINUE
* not. * otherwise.
*/ */
@Override @Override
public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) { public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) {
@ -1169,9 +1175,8 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
try { try {
/* /*
* Determine whether or not the file is an auto ingest job * Determine whether or not the file is a manifest file. If it
* manifest file. If it is, then parse it. Otherwise, move on to * is, then parse it.
* the next file in the directory.
*/ */
Manifest manifest = null; Manifest manifest = null;
for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) { for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) {
@ -1195,136 +1200,118 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
if (manifest == null) { if (manifest == null) {
return CONTINUE; return CONTINUE;
} }
/* /*
* If a manifest file has been found, get a manifest file lock, * If a manifest file has been found, get the corresponding auto
* analyze the job state, and put a job into the appropriate job * ingest job state from the manifest file coordination service
* list. There is a short wait here in case the input directory * node and put the job in the appropriate jobs list.
* scanner file visitor of another auto ingest node (AIN) has *
* the lock. If the lock ultmiately can't be obtained, the wait * There can be a race condition between queuing jobs and case
* was not long enough, or another auto ingest node (AIN) is * deletion. However, in practice eliminating the race condition
* holding the lock because it is executing the job, or a case * by acquiring a manifest file coordination service lock when
* deletion task has aquired the lock. In all of these cases the * analyzing job state here appears to have a significant
* manifest can be skipped for this scan. * performance cost for both input directory scanning and
* dequeuing jobs. Therefore, job state must be checked again
* during job dequeuing, while actually holding the lock, before
* executing the job.
*/ */
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString(), INPUT_SCAN_LOCKING_TIMEOUT_MINS, TimeUnit.MINUTES)) { String manifestFilePath = manifest.getFilePath().toString();
if (null != manifestLock) { byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestFilePath);
if (null != rawData && rawData.length > 0) {
/* AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData);
* Now that the lock has been acquired, make sure the AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus();
* manifest is still here. This is a way to resolve the switch (processingStatus) {
* race condition between this task and case deletion case PENDING:
* tasks without resorting to a protocol using locking addPendingJob(manifest, nodeData);
* of the input directory. break;
*/ case PROCESSING:
if (!filePath.toFile().exists()) { doRecoveryIfCrashed(manifest, nodeData);
return CONTINUE; break;
} case COMPLETED:
addCompletedJob(manifest, nodeData);
byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString()); break;
if (null != rawData && rawData.length > 0) { case DELETED:
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData); break;
AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus(); default:
switch (processingStatus) { sysLogger.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus");
case PENDING: break;
addPendingJob(manifest, nodeData);
break;
case PROCESSING:
/*
* If an exclusive manifest file lock was
* obtained for an auto ingest job in the
* processing state, the auto ingest node
* (AIN) executing the job crashed and the
* lock was released when the coordination
* service detected that the AIN was no
* longer alive.
*/
doCrashRecovery(manifest, nodeData);
break;
case COMPLETED:
addCompletedJob(manifest, nodeData);
break;
case DELETED:
/*
* Ignore jobs marked as deleted.
*/
break;
default:
sysLogger.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus");
break;
}
} else {
try {
addNewPendingJob(manifest);
} catch (AutoIngestJobException ex) {
sysLogger.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifest.getFilePath()), ex);
}
}
} }
} catch (CoordinationServiceException | AutoIngestJobException | AutoIngestJobNodeData.InvalidDataException ex) { } else {
sysLogger.log(Level.SEVERE, String.format("Error handling manifest at %s", manifest.getFilePath()), ex); addNewPendingJob(manifest);
} catch (InterruptedException ex) {
/*
* The thread running the input directory scan task was
* interrupted while blocked, i.e., auto ingest is shutting
* down.
*/
return TERMINATE;
} }
} catch (CoordinationServiceException | AutoIngestJobException | AutoIngestJobNodeData.InvalidDataException ex) {
sysLogger.log(Level.SEVERE, String.format("Error visiting %s", filePath), ex);
} catch (InterruptedException ex) {
return TERMINATE;
} catch (Exception ex) { } catch (Exception ex) {
/* /*
* This is an exception firewall so that an unexpected runtime * This is an exception firewall so that an unexpected runtime
* exception from the handling of a single manifest file does * exception from the handling of a single file does not stop
* not take out the input directory scanner. * the input directory scan.
*/ */
sysLogger.log(Level.SEVERE, String.format("Unexpected exception handling %s", filePath), ex); sysLogger.log(Level.SEVERE, String.format("Unexpected exception visiting %s", filePath), ex);
} }
if (!Thread.currentThread().isInterrupted()) { if (!Thread.currentThread().isInterrupted()) {
return CONTINUE; return CONTINUE;
} else { } else {
sysLogger.log(Level.WARNING, String.format("Auto ingest shut down while visiting %s", filePath));
return TERMINATE; return TERMINATE;
} }
} }
/** /**
* Adds an auto ingest job to the pending jobs queue. * Adds an existing auto ingest job to the pending jobs queue. If the
* version of the coordination service node data is out of date, it is
* upgraded to the current version.
* *
* @param manifest The manifest for the job. * @param manifest The manifest for the job.
* @param nodeData The data stored in the manifest file lock * @param nodeData The data stored in the manifest file coordination
* coordination service node for the job. * service node for the job.
* *
* @throws AutoIngestJobException If there was an error working * @throws AutoIngestJobException If there was an error working
* with the node data. * with the node data.
* @throws CoordinationServiceException If a lock node data version
* update was required and there
* was an error writing the node
* data by the coordination
* service.
* @throws InterruptedException If the thread running the input * @throws InterruptedException If the thread running the input
* directory scan task is * directory scan task is
* interrupted while blocked, i.e., * interrupted while blocked, i.e.,
* if auto ingest is shutting down. * if auto ingest is shutting down.
*/ */
private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws AutoIngestJobException, CoordinationServiceException, InterruptedException { private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws AutoIngestJobException, InterruptedException {
AutoIngestJob job; AutoIngestJob job;
if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) { if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) {
job = new AutoIngestJob(nodeData); job = new AutoIngestJob(nodeData);
} else { } else {
/*
* Upgrade the auto ingest node data to the current version.
*/
job = new AutoIngestJob(manifest); job = new AutoIngestJob(manifest);
job.setPriority(nodeData.getPriority()); job.setPriority(nodeData.getPriority());
Path caseDirectory = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); Path caseDirectory = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName());
if (null != caseDirectory) { if (null != caseDirectory) {
job.setCaseDirectoryPath(caseDirectory); job.setCaseDirectoryPath(caseDirectory);
} }
updateAutoIngestJobData(job);
/*
* Try to write the upgraded node data to coordination service
* manifest node data for the job. If the lock cannot be
* obtained, assume that the auto ingest node holding the lock
* is taking care of this.
*/
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) {
updateAutoIngestJobData(job);
}
} catch (CoordinationServiceException ex) {
sysLogger.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex);
}
} }
newPendingJobsList.add(job); newPendingJobsList.add(job);
} }
/** /**
* Adds a new job to the pending jobs queue. * Adds a new auto ingest job to the pending jobs queue.
* *
* @param manifest The manifest for the job. * @param manifest The manifest for the job.
* *
@ -1339,14 +1326,28 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
* if auto ingest is shutting down. * if auto ingest is shutting down.
*/ */
private void addNewPendingJob(Manifest manifest) throws AutoIngestJobException, CoordinationServiceException, InterruptedException { private void addNewPendingJob(Manifest manifest) throws AutoIngestJobException, CoordinationServiceException, InterruptedException {
AutoIngestJob job = new AutoIngestJob(manifest); /*
updateAutoIngestJobData(job); * Create the coordination service manifest file node data for the
newPendingJobsList.add(job); * job. Getting the lock both guards the writing of the new node
* data and creates the coordination service node if it does not
* already exist. Note that if this auto ingest node cannot get the
* lock, it is assumed that the auto ingest node holding the lock is
* taking care of this. In this case, this auto ingest node will not
* add the new job to its pending queue during this scan of the
* input directory, but it will be picked up during the next scan.
*/
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) {
AutoIngestJob job = new AutoIngestJob(manifest);
updateAutoIngestJobData(job);
newPendingJobsList.add(job);
}
}
} }
/** /**
* Does recovery for an auto ingest job that was left in the processing * If required, does recovery for an auto ingest job that was left in
* state by an auot ingest node (AIN) that crashed. * the processing state by an auto ingest node (AIN) that crashed.
* *
* @param manifest The manifest for the job. * @param manifest The manifest for the job.
* @param nodeData The data stored in the manifest file lock * @param nodeData The data stored in the manifest file lock
@ -1362,58 +1363,73 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
* interrupted while blocked, i.e., * interrupted while blocked, i.e.,
* if auto ingest is shutting down. * if auto ingest is shutting down.
*/ */
private void doCrashRecovery(Manifest manifest, AutoIngestJobNodeData jobNodeData) throws AutoIngestJobException, CoordinationServiceException, InterruptedException { private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData jobNodeData) throws AutoIngestJobException, CoordinationServiceException, InterruptedException {
String manifestPath = manifest.getFilePath().toString(); String manifestPath = manifest.getFilePath().toString();
sysLogger.log(Level.SEVERE, "Attempting crash recovery for {0}", manifestPath); try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifestPath)) {
AutoIngestJob job = new AutoIngestJob(jobNodeData); if (null != manifestLock) {
AutoIngestJob job = new AutoIngestJob(jobNodeData);
if (job.getProcessingStatus() == AutoIngestJob.ProcessingStatus.PROCESSING) {
/*
* If the lock can be obtained with the job status set
* to processing, then an auto ingest node crashed while
* executing the job and was unable to update the job
* status.
*/
sysLogger.log(Level.SEVERE, "Attempting crash recovery for {0}", manifestPath);
/* /*
* Try to set the error flags that indicate incomplete or messy data * First, try to set the case node data error flag that
* in displays for the job and the case. Note that if the job * indicates there was an auto ingest job error. If the
* crashed before a case directory was created, the job was a no-op, * auto ingest node that was executing the job crashed
* so the data quality flags do not need to be set. * before the case directory was created, the job was a
*/ * no-op, so the error flag does not need to be set.
Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); * However, note that if another auto ingest job
if (null != caseDirectoryPath) { * subsequently completed, the failed job may still have
job.setCaseDirectoryPath(caseDirectoryPath); * been a no-op, but in this case the flag will be set
job.setErrorsOccurred(true); * anyway, because a case directory will be found.
setCaseNodeDataErrorsOccurred(caseDirectoryPath); */
} else { Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName());
job.setErrorsOccurred(false); if (null != caseDirectoryPath) {
} job.setCaseDirectoryPath(caseDirectoryPath);
job.setErrorsOccurred(true);
setErrorsOccurredFlagForCase(caseDirectoryPath);
} else {
job.setErrorsOccurred(false);
}
/* /*
* Update the crash count for the job, determine whether or not to * Update the crash count for the job, determine whether
* retry processing its data source, and deal with the job * or not to retry processing its data source, and deal
* accordingly. * with the job accordingly.
*/ */
int numberOfCrashes = job.getNumberOfCrashes(); int numberOfCrashes = job.getNumberOfCrashes();
++numberOfCrashes; ++numberOfCrashes;
job.setNumberOfCrashes(numberOfCrashes); job.setNumberOfCrashes(numberOfCrashes);
if (numberOfCrashes < AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { if (numberOfCrashes < AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
job.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING); job.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING);
job.setCompletedDate(new Date(0)); job.setCompletedDate(new Date(0));
if (null != caseDirectoryPath) { if (null != caseDirectoryPath) {
try { try {
new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryWithRetry(); new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryWithRetry();
} catch (AutoIngestJobLoggerException ex) { } catch (AutoIngestJobLoggerException ex) {
sysLogger.log(Level.SEVERE, String.format("Error writing case auto ingest log entry for crashed job for %s", manifestPath), ex); sysLogger.log(Level.SEVERE, String.format("Error writing case auto ingest log entry for crashed job for %s", manifestPath), ex);
}
}
} else {
job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
job.setCompletedDate(Date.from(Instant.now()));
if (null != caseDirectoryPath) {
try {
new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryNoRetry();
} catch (AutoIngestJobLoggerException ex) {
sysLogger.log(Level.SEVERE, String.format("Error writing case auto ingest log entry for crashed job for %s", manifestPath), ex);
}
}
}
updateAutoIngestJobData(job);
newPendingJobsList.add(job);
} }
} }
updateAutoIngestJobData(job);
newPendingJobsList.add(job);
} else {
job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
job.setCompletedDate(Date.from(Instant.now()));
if (null != caseDirectoryPath) {
try {
new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), caseDirectoryPath).logCrashRecoveryNoRetry();
} catch (AutoIngestJobLoggerException ex) {
sysLogger.log(Level.SEVERE, String.format("Error writing case auto ingest log entry for crashed job for %s", manifestPath), ex);
}
}
updateAutoIngestJobData(job);
newCompletedJobsList.add(new AutoIngestJob(jobNodeData));
} }
} }
@ -1426,15 +1442,12 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
* *
* @throws AutoIngestJobException If there was an error working * @throws AutoIngestJobException If there was an error working
* with the node data. * with the node data.
* @throws CoordinationServiceException If there was an error writing
* updated node data by the
* coordination service.
* @throws InterruptedException If the thread running the input * @throws InterruptedException If the thread running the input
* directory scan task is * directory scan task is
* interrupted while blocked, i.e., * interrupted while blocked, i.e.,
* if auto ingest is shutting down. * if auto ingest is shutting down.
*/ */
private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws AutoIngestJobException, CoordinationServiceException, InterruptedException { private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws AutoIngestJobException, InterruptedException {
Path caseDirectoryPath = nodeData.getCaseDirectoryPath(); Path caseDirectoryPath = nodeData.getCaseDirectoryPath();
if (!caseDirectoryPath.toFile().exists()) { if (!caseDirectoryPath.toFile().exists()) {
sysLogger.log(Level.WARNING, String.format("Job completed for %s, but cannot find case directory %s, ignoring job", nodeData.getManifestFilePath(), caseDirectoryPath.toString())); sysLogger.log(Level.WARNING, String.format("Job completed for %s, but cannot find case directory %s, ignoring job", nodeData.getManifestFilePath(), caseDirectoryPath.toString()));
@ -1446,19 +1459,11 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
job = new AutoIngestJob(nodeData); job = new AutoIngestJob(nodeData);
job.setCaseDirectoryPath(caseDirectoryPath); job.setCaseDirectoryPath(caseDirectoryPath);
} else { } else {
/** /*
* Use the manifest rather than the node data here to create a * Upgrade the auto ingest node data to the current version.
* new AutoIngestJob instance because the AutoIngestJob
* constructor that takes a node data object expects the node
* data to have fields that do not exist in earlier versions.
*/ */
job = new AutoIngestJob(manifest); job = new AutoIngestJob(manifest);
job.setCaseDirectoryPath(caseDirectoryPath); job.setCaseDirectoryPath(caseDirectoryPath);
/**
* Update the job with the fields that exist in all versions of
* the nodeData.
*/
job.setCompletedDate(nodeData.getCompletedDate()); job.setCompletedDate(nodeData.getCompletedDate());
job.setErrorsOccurred(nodeData.getErrorsOccurred()); job.setErrorsOccurred(nodeData.getErrorsOccurred());
job.setPriority(nodeData.getPriority()); job.setPriority(nodeData.getPriority());
@ -1466,7 +1471,19 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
job.setProcessingStage(AutoIngestJob.Stage.COMPLETED, nodeData.getCompletedDate()); job.setProcessingStage(AutoIngestJob.Stage.COMPLETED, nodeData.getCompletedDate());
job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
updateAutoIngestJobData(job); /*
* Try to write the upgraded node data to coordination service
* manifest node data for the job. If the lock cannot be
* obtained, assume that the auto ingest node holding the lock
* is taking care of this.
*/
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) {
updateAutoIngestJobData(job);
}
} catch (CoordinationServiceException ex) {
sysLogger.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex);
}
} }
newCompletedJobsList.add(job); newCompletedJobsList.add(job);
@ -1964,6 +1981,17 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
} }
try { try {
/*
* There can be a race condition between queuing jobs
* and case deletion. However, in practice eliminating
* the race condition by acquiring a manifest file
* coordination service lock when analyzing job state
* during the input directory scan appears to have a
* significant performance cost for both input directory
* scanning and dequeuing jobs. Therefore, job state
* must be checked again here, while actually holding
* the lock, before executing the job.
*/
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString())); AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()));
if (!nodeData.getProcessingStatus().equals(PENDING)) { if (!nodeData.getProcessingStatus().equals(PENDING)) {
iterator.remove(); iterator.remove();
@ -2097,7 +2125,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
if (currentJob.isCanceled()) { if (currentJob.isCanceled()) {
Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath();
if (null != caseDirectoryPath) { if (null != caseDirectoryPath) {
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, currentJob.getManifest().getDataSourceFileName(), caseDirectoryPath); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, currentJob.getManifest().getDataSourceFileName(), caseDirectoryPath);
jobLogger.logJobCancelled(); jobLogger.logJobCancelled();
} }
@ -2455,7 +2483,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
if (!dataSource.exists()) { if (!dataSource.exists()) {
sysLogger.log(Level.SEVERE, "Missing data source for {0}", manifestPath); sysLogger.log(Level.SEVERE, "Missing data source for {0}", manifestPath);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logMissingDataSource(); jobLogger.logMissingDataSource();
return null; return null;
} }
@ -2500,7 +2528,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
// did we find a data source processor that can process the data source // did we find a data source processor that can process the data source
if (validDataSourceProcessors.isEmpty()) { if (validDataSourceProcessors.isEmpty()) {
// This should never happen. We should add all unsupported data sources as logical files. // This should never happen. We should add all unsupported data sources as logical files.
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
jobLogger.logFailedToIdentifyDataSource(); jobLogger.logFailedToIdentifyDataSource();
sysLogger.log(Level.WARNING, "Unsupported data source {0} for {1}", new Object[]{dataSource.getPath(), manifestPath}); // NON-NLS sysLogger.log(Level.WARNING, "Unsupported data source {0} for {1}", new Object[]{dataSource.getPath(), manifestPath}); // NON-NLS
@ -2535,7 +2563,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
// If we get to this point, none of the processors were successful // If we get to this point, none of the processors were successful
sysLogger.log(Level.SEVERE, "All data source processors failed to process {0}", dataSource.getPath()); sysLogger.log(Level.SEVERE, "All data source processors failed to process {0}", dataSource.getPath());
jobLogger.logFailedToAddDataSource(); jobLogger.logFailedToAddDataSource();
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
// Throw an exception. It will get caught & handled upstream and will result in AIM auto-pause. // Throw an exception. It will get caught & handled upstream and will result in AIM auto-pause.
throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Failed to process " + dataSource.getPath() + " with all data source processors"); throw new AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException("Failed to process " + dataSource.getPath() + " with all data source processors");
@ -2654,7 +2682,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
if (!cancelledModules.isEmpty()) { if (!cancelledModules.isEmpty()) {
sysLogger.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath)); sysLogger.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath));
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
for (String module : snapshot.getCancelledDataSourceIngestModules()) { for (String module : snapshot.getCancelledDataSourceIngestModules()) {
sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath));
nestedJobLogger.logIngestModuleCancelled(module); nestedJobLogger.logIngestModuleCancelled(module);
@ -2664,7 +2692,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
} else { } else {
currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now()));
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
nestedJobLogger.logAnalysisCancelled(); nestedJobLogger.logAnalysisCancelled();
CancellationReason cancellationReason = snapshot.getCancellationReason(); CancellationReason cancellationReason = snapshot.getCancellationReason();
if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) {
@ -2677,13 +2705,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable());
} }
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logIngestModuleStartupErrors(); jobLogger.logIngestModuleStartupErrors();
throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath));
} else { } else {
sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException());
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logAnalysisStartupError(); jobLogger.logAnalysisStartupError();
throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException()); throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException());
} }
@ -2692,7 +2720,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); sysLogger.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning});
} }
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logIngestJobSettingsErrors(); jobLogger.logIngestJobSettingsErrors();
throw new AnalysisStartupException("Error(s) in ingest job settings"); throw new AnalysisStartupException("Error(s) in ingest job settings");
} }
@ -2775,7 +2803,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
} catch (FileExportException ex) { } catch (FileExportException ex) {
sysLogger.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex); sysLogger.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
setCaseNodeDataErrorsOccurred(caseDirectoryPath); setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logFileExportError(); jobLogger.logFileExportError();
} }
} }