mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-14 17:06:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tracking auto ingest alerts via case nodes.

Browse Source
This commit is contained in:
U-BASIS\dgrove 2017-11-06 13:03:28 -05:00
parent 31e2e4ef9f
commit c95fa28c49
2 changed files with 111 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAlertFile.java
View File

@ -1,108 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import java.io.IOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.logging.Level;
/**
* Utility for creating and checking for the existence of an automated ingest
* alert file. The purpose of the file is to put a marker in the case directory
* when an error or warning occurs in connection with an automated ingest job.
*/
final class AutoIngestAlertFile {
private static final String ERROR_FILE_NAME = "autoingest.alert";
/**
* Checks whether an automated ingest alert file exists in a case directory.
*
* @param caseDirectoryPath The case directory path.
*
* @return True or false.
*/
static boolean exists(Path caseDirectoryPath) {
return caseDirectoryPath.resolve(ERROR_FILE_NAME).toFile().exists();
}
/**
* Creates an automated ingest alert file in a case directory if such a file
* does not already exist.
*
* @param caseDirectoryPath The case directory path.
*
* @return True or false.
*/
static void create(Path caseDirectoryPath) throws AutoIngestAlertFileException {
try {
Files.createFile(caseDirectoryPath.resolve(ERROR_FILE_NAME));
} catch (FileAlreadyExistsException ignored) {
/*
* The file already exists, the exception is not exceptional.
*/
} catch (IOException ex) {
/*
* FileAlreadyExistsException implementation is optional, so check
* for that case.
*/
if (!exists(caseDirectoryPath)) {
throw new AutoIngestAlertFileException(String.format("Error creating automated ingest alert file in %s", caseDirectoryPath), ex);
}
}
}
/**
* Exception thrown when there is a problem creating an alert file.
*/
final static class AutoIngestAlertFileException extends Exception {
private static final long serialVersionUID = 1L;
/**
* Constructs an exception to throw when there is a problem creating an
* alert file.
*
* @param message The exception message.
*/
private AutoIngestAlertFileException(String message) {
super(message);
}
/**
* Constructs an exception to throw when there is a problem creating an
* alert file.
*
* @param message The exception message.
* @param cause The cause of the exception, if it was an exception.
*/
private AutoIngestAlertFileException(String message, Throwable cause) {
super(message, cause);
}
}
/**
* Prevents instantiation of this utility class.
*/
private AutoIngestAlertFile() {
}
}

198
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

@ -64,6 +64,7 @@ import org.sleuthkit.autopsy.casemodule.Case.CaseType;
import org.sleuthkit.autopsy.casemodule.CaseActionException; import org.sleuthkit.autopsy.casemodule.CaseActionException;
import org.sleuthkit.autopsy.casemodule.CaseDetails; import org.sleuthkit.autopsy.casemodule.CaseDetails;
import org.sleuthkit.autopsy.casemodule.CaseMetadata; import org.sleuthkit.autopsy.casemodule.CaseMetadata;
import org.sleuthkit.autopsy.casemodule.CaseNodeData;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService.Lock; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.Lock;
@ -78,7 +79,6 @@ import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.events.AutopsyEvent;
import org.sleuthkit.autopsy.events.AutopsyEventException; import org.sleuthkit.autopsy.events.AutopsyEventException;
import org.sleuthkit.autopsy.events.AutopsyEventPublisher; import org.sleuthkit.autopsy.events.AutopsyEventPublisher;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestAlertFile.AutoIngestAlertFileException;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJobLogger.AutoIngestJobLoggerException; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJobLogger.AutoIngestJobLoggerException;
import org.sleuthkit.autopsy.experimental.autoingest.FileExporter.FileExportException; import org.sleuthkit.autopsy.experimental.autoingest.FileExporter.FileExportException;
import org.sleuthkit.autopsy.experimental.autoingest.ManifestFileParser.ManifestFileParserException; import org.sleuthkit.autopsy.experimental.autoingest.ManifestFileParser.ManifestFileParserException;
@ -98,6 +98,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestJobStartResult;
import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.IngestModuleError; import org.sleuthkit.autopsy.ingest.IngestModuleError;
import org.sleuthkit.datamodel.Content;
/** /**
* An auto ingest manager is responsible for processing auto ingest jobs defined * An auto ingest manager is responsible for processing auto ingest jobs defined
@ -554,7 +555,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
++maxPriority; ++maxPriority;
for (AutoIngestJob job : prioritizedJobs) { for (AutoIngestJob job : prioritizedJobs) {
try { try {
this.updateCoordinationServiceNode(job); this.updateCoordinationServiceManifestNode(job);
job.setPriority(maxPriority); job.setPriority(maxPriority);
} catch (CoordinationServiceException | InterruptedException ex) { } catch (CoordinationServiceException | InterruptedException ex) {
throw new AutoIngestManagerException("Error updating case priority", ex); throw new AutoIngestManagerException("Error updating case priority", ex);
@ -602,13 +603,13 @@ public final class AutoIngestManager extends Observable implements PropertyChang
} }
/* /*
* Bump the priority by one and update the coordination service node * Bump the priority by one and update the coordination service
* data for the job. * manifest node data for the job.
*/ */
if (null != prioritizedJob) { if (null != prioritizedJob) {
++maxPriority; ++maxPriority;
try { try {
this.updateCoordinationServiceNode(prioritizedJob); this.updateCoordinationServiceManifestNode(prioritizedJob);
} catch (CoordinationServiceException | InterruptedException ex) { } catch (CoordinationServiceException | InterruptedException ex) {
throw new AutoIngestManagerException("Error updating job priority", ex); throw new AutoIngestManagerException("Error updating job priority", ex);
} }
@ -649,7 +650,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
/* /*
* Add the job to the pending jobs queue and update the coordination * Add the job to the pending jobs queue and update the coordination
* service node data for the job. * service manifest node data for the job.
*/ */
if (null != completedJob && !completedJob.getCaseDirectoryPath().toString().isEmpty()) { if (null != completedJob && !completedJob.getCaseDirectoryPath().toString().isEmpty()) {
try { try {
@ -661,7 +662,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
completedJob.setCompletedDate(new Date(0)); completedJob.setCompletedDate(new Date(0));
completedJob.setProcessingStatus(PENDING); completedJob.setProcessingStatus(PENDING);
completedJob.setProcessingStage(AutoIngestJob.Stage.PENDING, Date.from(Instant.now())); completedJob.setProcessingStage(AutoIngestJob.Stage.PENDING, Date.from(Instant.now()));
updateCoordinationServiceNode(completedJob); updateCoordinationServiceManifestNode(completedJob);
pendingJobs.add(completedJob); pendingJobs.add(completedJob);
} catch (CoordinationServiceException ex) { } catch (CoordinationServiceException ex) {
SYS_LOGGER.log(Level.SEVERE, String.format("Coordination service error while reprocessing %s", manifestPath), ex); SYS_LOGGER.log(Level.SEVERE, String.format("Coordination service error while reprocessing %s", manifestPath), ex);
@ -755,7 +756,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString())); AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()));
AutoIngestJob deletedJob = new AutoIngestJob(nodeData); AutoIngestJob deletedJob = new AutoIngestJob(nodeData);
deletedJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.DELETED); deletedJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.DELETED);
this.updateCoordinationServiceNode(deletedJob); this.updateCoordinationServiceManifestNode(deletedJob);
} catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) {
SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex);
return CaseDeletionResult.PARTIALLY_DELETED; return CaseDeletionResult.PARTIALLY_DELETED;
@ -865,21 +866,32 @@ public final class AutoIngestManager extends Observable implements PropertyChang
} }
/** /**
* Sets the coordination service node data for an auto ingest job. * Sets the coordination service manifest node.
* *
* Note that a new auto ingest node data object will be created from the job * Note that a new auto ingest job node data object will be created from
* passed in. Thus, if the data version of the node has changed, the node * the job passed in. Thus, if the data version of the node has changed,
* will be "upgraded" as well as updated. * the node will be "upgraded" as well as updated.
* *
* @param job The auto ingest job. * @param job The auto ingest job.
*/ */
void updateCoordinationServiceNode(AutoIngestJob job) throws CoordinationServiceException, InterruptedException { void updateCoordinationServiceManifestNode(AutoIngestJob job) throws CoordinationServiceException, InterruptedException {
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(job); AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(job);
String manifestNodePath = job.getManifest().getFilePath().toString(); String manifestNodePath = job.getManifest().getFilePath().toString();
byte[] rawData = nodeData.toArray(); byte[] rawData = nodeData.toArray();
coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestNodePath, rawData); coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestNodePath, rawData);
} }
/**
* Sets the coordination service case node.
*
* @param caseNodeData The case node data.
* @param caseDirectoryPath The case directory.
*/
void updateCoordinationServiceCaseNode(CaseNodeData caseNodeData, Path caseDirectoryPath) throws CoordinationServiceException, InterruptedException {
byte[] rawData = caseNodeData.toArray();
coordinationService.setNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString(), rawData);
}
/** /**
* A task that submits an input directory scan task to the input directory * A task that submits an input directory scan task to the input directory
* scan task executor. * scan task executor.
@ -1147,8 +1159,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang
} }
/* /*
* Try to upgrade/update the coordination service node data for * Try to upgrade/update the coordination service manifest node
* the job. * data for the job.
* *
* An exclusive lock is obtained before doing so because another * An exclusive lock is obtained before doing so because another
* host may have already found the job, obtained an exclusive * host may have already found the job, obtained an exclusive
@ -1161,7 +1173,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
*/ */
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) { if (null != manifestLock) {
updateCoordinationServiceNode(job); updateCoordinationServiceManifestNode(job);
} }
} catch (CoordinationServiceException ex) { } catch (CoordinationServiceException ex) {
SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex);
@ -1186,9 +1198,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
*/ */
private void addNewPendingJob(Manifest manifest) throws InterruptedException, AutoIngestJobException { private void addNewPendingJob(Manifest manifest) throws InterruptedException, AutoIngestJobException {
/* /*
* Create the coordination service node data for the job. Note that * Create the coordination service manifest node data for the job.
* getting the lock will create the node for the job (with no data) * Note that getting the lock will create the node for the job
* if it does not already exist. * (with no data) if it does not already exist.
* *
* An exclusive lock is obtained before creating the node data * An exclusive lock is obtained before creating the node data
* because another host may have already found the job, obtained an * because another host may have already found the job, obtained an
@ -1202,7 +1214,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) { if (null != manifestLock) {
AutoIngestJob job = new AutoIngestJob(manifest); AutoIngestJob job = new AutoIngestJob(manifest);
updateCoordinationServiceNode(job); updateCoordinationServiceManifestNode(job);
newPendingJobsList.add(job); newPendingJobsList.add(job);
} }
} catch (CoordinationServiceException ex) { } catch (CoordinationServiceException ex) {
@ -1219,14 +1231,14 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* status was not updated. * status was not updated.
* *
* @param manifest The manifest for upgrading the node. * @param manifest The manifest for upgrading the node.
* @param nodeData The node data. * @param jobNodeData The auto ingest job node data.
* *
* @throws InterruptedException if the thread running the input * @throws InterruptedException if the thread running the input
* directory scan task is interrupted while * directory scan task is interrupted while
* blocked, i.e., if auto ingest is * blocked, i.e., if auto ingest is
* shutting down. * shutting down.
*/ */
private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException { private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData jobNodeData) throws InterruptedException, AutoIngestJobException {
/* /*
* Try to get an exclusive lock on the coordination service node for * Try to get an exclusive lock on the coordination service node for
* the job. If the lock cannot be obtained, another host in the auto * the job. If the lock cannot be obtained, another host in the auto
@ -1240,7 +1252,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
/* /*
* Create the recovery job. * Create the recovery job.
*/ */
AutoIngestJob job = new AutoIngestJob(nodeData); AutoIngestJob job = new AutoIngestJob(jobNodeData);
int numberOfCrashes = job.getNumberOfCrashes(); int numberOfCrashes = job.getNumberOfCrashes();
++numberOfCrashes; ++numberOfCrashes;
job.setNumberOfCrashes(numberOfCrashes); job.setNumberOfCrashes(numberOfCrashes);
@ -1254,15 +1266,16 @@ public final class AutoIngestManager extends Observable implements PropertyChang
} }
/* /*
* Update the coordination service node for the job. If * Update the coordination service manifest node for
* this fails, leave the recovery to another host. * the job. If this fails, leave the recovery to
* another host.
*/ */
try { try {
updateCoordinationServiceNode(job); updateCoordinationServiceManifestNode(job);
if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
newPendingJobsList.add(job); newPendingJobsList.add(job);
} else { } else {
newCompletedJobsList.add(new AutoIngestJob(nodeData)); newCompletedJobsList.add(new AutoIngestJob(jobNodeData));
} }
} catch (CoordinationServiceException ex) { } catch (CoordinationServiceException ex) {
SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex); SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex);
@ -1270,13 +1283,15 @@ public final class AutoIngestManager extends Observable implements PropertyChang
} }
/* /*
* Write the alert file and do the logging. * Update the case node data and do the logging.
*/ */
if (null != caseDirectoryPath) { if (null != caseDirectoryPath) {
try { try {
AutoIngestAlertFile.create(nodeData.getCaseDirectoryPath()); CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
} catch (AutoIngestAlertFileException ex) { caseNodeData.setErrorsOccurred(true);
SYS_LOGGER.log(Level.SEVERE, String.format("Error creating alert file for crashed job for %s", manifestPath), ex); updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
} catch (CaseNodeData.InvalidDataException ex) {
SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to get case node data for %s", caseDirectoryPath), ex);
} }
} }
if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) {
@ -1292,7 +1307,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
if (null != caseDirectoryPath) { if (null != caseDirectoryPath) {
try { try {
new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), nodeData.getCaseDirectoryPath()).logCrashRecoveryNoRetry(); new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), jobNodeData.getCaseDirectoryPath()).logCrashRecoveryNoRetry();
} catch (AutoIngestJobLoggerException ex) { } catch (AutoIngestJobLoggerException ex) {
SYS_LOGGER.log(Level.SEVERE, String.format("Error creating case auto ingest log entry for crashed job for %s", manifestPath), ex); SYS_LOGGER.log(Level.SEVERE, String.format("Error creating case auto ingest log entry for crashed job for %s", manifestPath), ex);
} }
@ -1352,15 +1367,16 @@ public final class AutoIngestManager extends Observable implements PropertyChang
job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED);
/* /*
* Try to upgrade/update the coordination service node data * Try to upgrade/update the coordination service manifest
* for the job. It is possible that two hosts will both try * node data for the job. It is possible that two hosts
* to obtain the lock to do the upgrade operation at the * will both try to obtain the lock to do the upgrade
* same time. If this happens, the host that is holding the * operation at the same time. If this happens, the host
* lock will complete the upgrade operation. * that is holding the lock will complete the upgrade
* operation.
*/ */
try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) {
if (null != manifestLock) { if (null != manifestLock) {
updateCoordinationServiceNode(job); updateCoordinationServiceManifestNode(job);
} }
} catch (CoordinationServiceException ex) { } catch (CoordinationServiceException ex) {
SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex);
@ -1507,8 +1523,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
errorState = ErrorState.ANALYSIS_STARTUP_ERROR; errorState = ErrorState.ANALYSIS_STARTUP_ERROR;
} else if (ex instanceof FileExportException) { } else if (ex instanceof FileExportException) {
errorState = ErrorState.FILE_EXPORT_ERROR; errorState = ErrorState.FILE_EXPORT_ERROR;
} else if (ex instanceof AutoIngestAlertFileException) {
errorState = ErrorState.ALERT_FILE_ERROR;
} else if (ex instanceof AutoIngestJobLoggerException) { } else if (ex instanceof AutoIngestJobLoggerException) {
errorState = ErrorState.JOB_LOGGER_ERROR; errorState = ErrorState.JOB_LOGGER_ERROR;
} else if (ex instanceof AutoIngestDataSourceProcessorException) { } else if (ex instanceof AutoIngestDataSourceProcessorException) {
@ -1691,9 +1705,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* @throws FileExportException if there is an * @throws FileExportException if there is an
* error exporting * error exporting
* files. * files.
* @throws AutoIngestAlertFileException if there is an
* error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an * @throws AutoIngestJobLoggerException if there is an
* error writing to * error writing to
* the auto ingest * the auto ingest
@ -1710,7 +1721,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* auto ingest node * auto ingest node
* data objects. * data objects.
*/ */
private void processJobs() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, AutoIngestJobNodeData.InvalidDataException { private void processJobs() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, AutoIngestJobNodeData.InvalidDataException, CaseNodeData.InvalidDataException {
SYS_LOGGER.log(Level.INFO, "Started processing pending jobs queue"); SYS_LOGGER.log(Level.INFO, "Started processing pending jobs queue");
Lock manifestLock = JobProcessingTask.this.dequeueAndLockNextJob(); Lock manifestLock = JobProcessingTask.this.dequeueAndLockNextJob();
while (null != manifestLock) { while (null != manifestLock) {
@ -1890,9 +1901,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* @throws FileExportException if there is an * @throws FileExportException if there is an
* error exporting * error exporting
* files. * files.
* @throws AutoIngestAlertFileException if there is an
* error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an * @throws AutoIngestJobLoggerException if there is an
* error writing to * error writing to
* the auto ingest * the auto ingest
@ -1909,13 +1917,13 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* auto ingest node * auto ingest node
* data objects. * data objects.
*/ */
private void processJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, AutoIngestJobNodeData.InvalidDataException { private void processJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException {
Path manifestPath = currentJob.getManifest().getFilePath(); Path manifestPath = currentJob.getManifest().getFilePath();
SYS_LOGGER.log(Level.INFO, "Started processing of {0}", manifestPath); SYS_LOGGER.log(Level.INFO, "Started processing of {0}", manifestPath);
currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PROCESSING); currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PROCESSING);
currentJob.setProcessingStage(AutoIngestJob.Stage.STARTING, Date.from(Instant.now())); currentJob.setProcessingStage(AutoIngestJob.Stage.STARTING, Date.from(Instant.now()));
currentJob.setProcessingHostName(AutoIngestManager.LOCAL_HOST_NAME); currentJob.setProcessingHostName(AutoIngestManager.LOCAL_HOST_NAME);
updateCoordinationServiceNode(currentJob); updateCoordinationServiceManifestNode(currentJob);
setChanged(); setChanged();
notifyObservers(Event.JOB_STARTED); notifyObservers(Event.JOB_STARTED);
eventPublisher.publishRemotely(new AutoIngestJobStartedEvent(currentJob)); eventPublisher.publishRemotely(new AutoIngestJobStartedEvent(currentJob));
@ -1939,14 +1947,16 @@ public final class AutoIngestManager extends Observable implements PropertyChang
currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING); currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING);
} }
currentJob.setProcessingHostName(""); currentJob.setProcessingHostName("");
updateCoordinationServiceNode(currentJob); updateCoordinationServiceManifestNode(currentJob);
boolean retry = (!currentJob.isCanceled() && !currentJob.isCompleted()); boolean retry = (!currentJob.isCanceled() && !currentJob.isCompleted());
SYS_LOGGER.log(Level.INFO, "Completed processing of {0}, retry = {1}", new Object[]{manifestPath, retry}); SYS_LOGGER.log(Level.INFO, "Completed processing of {0}, retry = {1}", new Object[]{manifestPath, retry});
if (currentJob.isCanceled()) { if (currentJob.isCanceled()) {
Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath();
if (null != caseDirectoryPath) { if (null != caseDirectoryPath) {
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, currentJob.getManifest().getDataSourceFileName(), caseDirectoryPath); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, currentJob.getManifest().getDataSourceFileName(), caseDirectoryPath);
jobLogger.logJobCancelled(); jobLogger.logJobCancelled();
} }
@ -1994,7 +2004,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* i.e., if auto ingest is * i.e., if auto ingest is
* shutting down. * shutting down.
*/ */
private void attemptJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { private void attemptJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException {
updateConfiguration(); updateConfiguration();
if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) { if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) {
return; return;
@ -2159,8 +2169,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* level ingest modules. * level ingest modules.
* @throws FileExportException if there is an error exporting * @throws FileExportException if there is an error exporting
* files. * files.
* @throws AutoIngestAlertFileException if there is an error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an error writing to * @throws AutoIngestJobLoggerException if there is an error writing to
* the auto ingest log for the * the auto ingest log for the
* case. * case.
@ -2169,7 +2177,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* while blocked, i.e., if auto * while blocked, i.e., if auto
* ingest is shutting down. * ingest is shutting down.
*/ */
private void runIngestForJob(Case caseForJob) throws CoordinationServiceException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { private void runIngestForJob(Case caseForJob) throws CoordinationServiceException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException {
try { try {
if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) { if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) {
return; return;
@ -2197,8 +2205,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* level ingest modules. * level ingest modules.
* @throws FileExportException if there is an error exporting * @throws FileExportException if there is an error exporting
* files. * files.
* @throws AutoIngestAlertFileException if there is an error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an error writing to * @throws AutoIngestJobLoggerException if there is an error writing to
* the auto ingest log for the * the auto ingest log for the
* case. * case.
@ -2207,7 +2213,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* while blocked, i.e., if auto * while blocked, i.e., if auto
* ingest is shutting down. * ingest is shutting down.
*/ */
private void ingestDataSource(Case caseForJob) throws AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { private void ingestDataSource(Case caseForJob) throws AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException, CoordinationServiceException {
if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) { if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) {
return; return;
} }
@ -2255,8 +2261,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* *
* @return A data source object. * @return A data source object.
* *
* @throws AutoIngestAlertFileException if there is an error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an error writing to * @throws AutoIngestJobLoggerException if there is an error writing to
* the auto ingest log for the * the auto ingest log for the
* case. * case.
@ -2265,7 +2269,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* interrupted while blocked, i.e., * interrupted while blocked, i.e.,
* if auto ingest is shutting down. * if auto ingest is shutting down.
*/ */
private DataSource identifyDataSource() throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { private DataSource identifyDataSource() throws AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException {
Manifest manifest = currentJob.getManifest(); Manifest manifest = currentJob.getManifest();
Path manifestPath = manifest.getFilePath(); Path manifestPath = manifest.getFilePath();
SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath); SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath);
@ -2277,7 +2281,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
if (!dataSource.exists()) { if (!dataSource.exists()) {
SYS_LOGGER.log(Level.SEVERE, "Missing data source for {0}", manifestPath); SYS_LOGGER.log(Level.SEVERE, "Missing data source for {0}", manifestPath);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logMissingDataSource(); jobLogger.logMissingDataSource();
return null; return null;
} }
@ -2291,8 +2297,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* *
* @param dataSource The data source. * @param dataSource The data source.
* *
* @throws AutoIngestAlertFileException if there is an error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an error writing to * @throws AutoIngestJobLoggerException if there is an error writing to
* the auto ingest log for the * the auto ingest log for the
* case. * case.
@ -2301,7 +2305,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* while blocked, i.e., if auto * while blocked, i.e., if auto
* ingest is shutting down. * ingest is shutting down.
*/ */
private void runDataSourceProcessor(Case caseForJob, DataSource dataSource) throws InterruptedException, AutoIngestAlertFileException, AutoIngestJobLoggerException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { private void runDataSourceProcessor(Case caseForJob, DataSource dataSource) throws InterruptedException, AutoIngestJobLoggerException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException, CoordinationServiceException {
Manifest manifest = currentJob.getManifest(); Manifest manifest = currentJob.getManifest();
Path manifestPath = manifest.getFilePath(); Path manifestPath = manifest.getFilePath();
SYS_LOGGER.log(Level.INFO, "Adding data source for {0} ", manifestPath); SYS_LOGGER.log(Level.INFO, "Adding data source for {0} ", manifestPath);
@ -2327,7 +2331,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
// did we find a data source processor that can process the data source // did we find a data source processor that can process the data source
if (validDataSourceProcessorsMap.isEmpty()) { if (validDataSourceProcessorsMap.isEmpty()) {
// This should never happen. We should add all unsupported data sources as logical files. // This should never happen. We should add all unsupported data sources as logical files.
AutoIngestAlertFile.create(caseDirectoryPath); CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
jobLogger.logFailedToIdentifyDataSource(); jobLogger.logFailedToIdentifyDataSource();
SYS_LOGGER.log(Level.WARNING, "Unsupported data source {0} for {1}", new Object[]{dataSource.getPath(), manifestPath}); // NON-NLS SYS_LOGGER.log(Level.WARNING, "Unsupported data source {0} for {1}", new Object[]{dataSource.getPath(), manifestPath}); // NON-NLS
@ -2353,7 +2359,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
// Log that the current DSP failed and set the error flag. We consider it an error // Log that the current DSP failed and set the error flag. We consider it an error
// if a DSP fails even if a later one succeeds since we expected to be able to process // if a DSP fails even if a later one succeeds since we expected to be able to process
// the data source which each DSP on the list. // the data source which each DSP on the list.
AutoIngestAlertFile.create(caseDirectoryPath); CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
jobLogger.logDataSourceProcessorError(selectedProcessor.getDataSourceType()); jobLogger.logDataSourceProcessorError(selectedProcessor.getDataSourceType());
SYS_LOGGER.log(Level.SEVERE, "Exception while processing {0} with data source processor {1}", new Object[]{dataSource.getPath(), selectedProcessor.getDataSourceType()}); SYS_LOGGER.log(Level.SEVERE, "Exception while processing {0} with data source processor {1}", new Object[]{dataSource.getPath(), selectedProcessor.getDataSourceType()});
@ -2377,8 +2385,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* *
* @param dataSource The data source. * @param dataSource The data source.
* *
* @throws AutoIngestAlertFileException if there is an error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an error writing to * @throws AutoIngestJobLoggerException if there is an error writing to
* the auto ingest log for the * the auto ingest log for the
* case. * case.
@ -2387,7 +2393,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* while blocked, i.e., if auto * while blocked, i.e., if auto
* ingest is shutting down. * ingest is shutting down.
*/ */
private void logDataSourceProcessorResult(DataSource dataSource) throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { private void logDataSourceProcessorResult(DataSource dataSource) throws AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException {
Manifest manifest = currentJob.getManifest(); Manifest manifest = currentJob.getManifest();
Path manifestPath = manifest.getFilePath(); Path manifestPath = manifest.getFilePath();
Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath();
@ -2399,7 +2405,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
jobLogger.logDataSourceAdded(); jobLogger.logDataSourceAdded();
if (dataSource.getContent().isEmpty()) { if (dataSource.getContent().isEmpty()) {
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logNoDataSourceContent(); jobLogger.logNoDataSourceContent();
} }
break; break;
@ -2411,7 +2419,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
jobLogger.logDataSourceAdded(); jobLogger.logDataSourceAdded();
if (dataSource.getContent().isEmpty()) { if (dataSource.getContent().isEmpty()) {
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logNoDataSourceContent(); jobLogger.logNoDataSourceContent();
} }
break; break;
@ -2421,7 +2431,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
SYS_LOGGER.log(Level.SEVERE, "Critical error running data source processor for {0}: {1}", new Object[]{manifestPath, errorMessage}); SYS_LOGGER.log(Level.SEVERE, "Critical error running data source processor for {0}: {1}", new Object[]{manifestPath, errorMessage});
} }
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logFailedToAddDataSource(); jobLogger.logFailedToAddDataSource();
break; break;
} }
@ -2435,7 +2447,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
*/ */
SYS_LOGGER.log(Level.WARNING, "Cancellation while waiting for data source processor for {0}", manifestPath); SYS_LOGGER.log(Level.WARNING, "Cancellation while waiting for data source processor for {0}", manifestPath);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logDataSourceProcessorCancelled(); jobLogger.logDataSourceProcessorCancelled();
} }
} }
@ -2449,8 +2463,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* *
* @throws AnalysisStartupException if there is an error analyzing * @throws AnalysisStartupException if there is an error analyzing
* the data source. * the data source.
* @throws AutoIngestAlertFileException if there is an error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an error writing to * @throws AutoIngestJobLoggerException if there is an error writing to
* the auto ingest log for the * the auto ingest log for the
* case. * case.
@ -2459,7 +2471,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* while blocked, i.e., if auto * while blocked, i.e., if auto
* ingest is shutting down. * ingest is shutting down.
*/ */
private void analyze(DataSource dataSource) throws AnalysisStartupException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { private void analyze(DataSource dataSource) throws AnalysisStartupException, AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException {
Manifest manifest = currentJob.getManifest(); Manifest manifest = currentJob.getManifest();
Path manifestPath = manifest.getFilePath(); Path manifestPath = manifest.getFilePath();
SYS_LOGGER.log(Level.INFO, "Starting ingest modules analysis for {0} ", manifestPath); SYS_LOGGER.log(Level.INFO, "Starting ingest modules analysis for {0} ", manifestPath);
@ -2491,7 +2503,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
if (!cancelledModules.isEmpty()) { if (!cancelledModules.isEmpty()) {
SYS_LOGGER.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath)); SYS_LOGGER.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath));
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
for (String module : snapshot.getCancelledDataSourceIngestModules()) { for (String module : snapshot.getCancelledDataSourceIngestModules()) {
SYS_LOGGER.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); SYS_LOGGER.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath));
jobLogger.logIngestModuleCancelled(module); jobLogger.logIngestModuleCancelled(module);
@ -2501,7 +2515,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
} else { } else {
currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now()));
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logAnalysisCancelled(); jobLogger.logAnalysisCancelled();
CancellationReason cancellationReason = snapshot.getCancellationReason(); CancellationReason cancellationReason = snapshot.getCancellationReason();
if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) {
@ -2514,13 +2530,17 @@ public final class AutoIngestManager extends Observable implements PropertyChang
SYS_LOGGER.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); SYS_LOGGER.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable());
} }
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logIngestModuleStartupErrors(); jobLogger.logIngestModuleStartupErrors();
throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath));
} else { } else {
SYS_LOGGER.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); SYS_LOGGER.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException());
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logAnalysisStartupError(); jobLogger.logAnalysisStartupError();
throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException()); throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException());
} }
@ -2529,7 +2549,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
SYS_LOGGER.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); SYS_LOGGER.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning});
} }
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logIngestJobSettingsErrors(); jobLogger.logIngestJobSettingsErrors();
throw new AnalysisStartupException("Error(s) in ingest job settings"); throw new AnalysisStartupException("Error(s) in ingest job settings");
} }
@ -2548,8 +2570,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* *
* @throws FileExportException if there is an error exporting * @throws FileExportException if there is an error exporting
* the files. * the files.
* @throws AutoIngestAlertFileException if there is an error creating an
* alert file.
* @throws AutoIngestJobLoggerException if there is an error writing to * @throws AutoIngestJobLoggerException if there is an error writing to
* the auto ingest log for the * the auto ingest log for the
* case. * case.
@ -2558,7 +2578,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
* while blocked, i.e., if auto * while blocked, i.e., if auto
* ingest is shutting down. * ingest is shutting down.
*/ */
private void exportFiles(DataSource dataSource) throws FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { private void exportFiles(DataSource dataSource) throws FileExportException, AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException {
Manifest manifest = currentJob.getManifest(); Manifest manifest = currentJob.getManifest();
Path manifestPath = manifest.getFilePath(); Path manifestPath = manifest.getFilePath();
SYS_LOGGER.log(Level.INFO, "Exporting files for {0}", manifestPath); SYS_LOGGER.log(Level.INFO, "Exporting files for {0}", manifestPath);
@ -2574,7 +2594,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang
} catch (FileExportException ex) { } catch (FileExportException ex) {
SYS_LOGGER.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex); SYS_LOGGER.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex);
currentJob.setErrorsOccurred(true); currentJob.setErrorsOccurred(true);
AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString()));
caseNodeData.setErrorsOccurred(true);
updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath);
jobLogger.logFileExportError(); jobLogger.logFileExportError();
} }
} }
@ -2740,7 +2762,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang
currentJob.getProcessingStageDetails(); currentJob.getProcessingStageDetails();
setChanged(); setChanged();
notifyObservers(Event.JOB_STATUS_UPDATED); notifyObservers(Event.JOB_STATUS_UPDATED);
updateCoordinationServiceNode(currentJob); updateCoordinationServiceManifestNode(currentJob);
eventPublisher.publishRemotely(new AutoIngestJobStatusEvent(currentJob)); eventPublisher.publishRemotely(new AutoIngestJobStatusEvent(currentJob));
} }
@ -2843,7 +2865,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
CASE_MANAGEMENT_ERROR("Case management error"), CASE_MANAGEMENT_ERROR("Case management error"),
ANALYSIS_STARTUP_ERROR("Analysis startup error"), ANALYSIS_STARTUP_ERROR("Analysis startup error"),
FILE_EXPORT_ERROR("File export error"), FILE_EXPORT_ERROR("File export error"),
ALERT_FILE_ERROR("Alert file error"),
JOB_LOGGER_ERROR("Job logger error"), JOB_LOGGER_ERROR("Job logger error"),
DATA_SOURCE_PROCESSOR_ERROR("Data source processor error"), DATA_SOURCE_PROCESSOR_ERROR("Data source processor error"),
UNEXPECTED_EXCEPTION("Unknown error"); UNEXPECTED_EXCEPTION("Unknown error");
@ -2918,7 +2939,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang
PARTIALLY_DELETED, PARTIALLY_DELETED,
FULLY_DELETED FULLY_DELETED
} }
static final class AutoIngestManagerException extends Exception { static final class AutoIngestManagerException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;