From c95fa28c4942da407a0596310df80a6c929eab5c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 6 Nov 2017 13:03:28 -0500 Subject: [PATCH] Tracking auto ingest alerts via case nodes. --- .../autoingest/AutoIngestAlertFile.java | 108 ---------- .../autoingest/AutoIngestManager.java | 202 ++++++++++-------- 2 files changed, 111 insertions(+), 199 deletions(-) delete mode 100755 Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAlertFile.java diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAlertFile.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAlertFile.java deleted file mode 100755 index 40fff351eb..0000000000 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAlertFile.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2015 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.experimental.autoingest; - -import java.io.IOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.logging.Level; - -/** - * Utility for creating and checking for the existence of an automated ingest - * alert file. The purpose of the file is to put a marker in the case directory - * when an error or warning occurs in connection with an automated ingest job. - */ -final class AutoIngestAlertFile { - - private static final String ERROR_FILE_NAME = "autoingest.alert"; - - /** - * Checks whether an automated ingest alert file exists in a case directory. - * - * @param caseDirectoryPath The case directory path. - * - * @return True or false. - */ - static boolean exists(Path caseDirectoryPath) { - return caseDirectoryPath.resolve(ERROR_FILE_NAME).toFile().exists(); - } - - /** - * Creates an automated ingest alert file in a case directory if such a file - * does not already exist. - * - * @param caseDirectoryPath The case directory path. - * - * @return True or false. - */ - static void create(Path caseDirectoryPath) throws AutoIngestAlertFileException { - try { - Files.createFile(caseDirectoryPath.resolve(ERROR_FILE_NAME)); - } catch (FileAlreadyExistsException ignored) { - /* - * The file already exists, the exception is not exceptional. - */ - } catch (IOException ex) { - /* - * FileAlreadyExistsException implementation is optional, so check - * for that case. - */ - if (!exists(caseDirectoryPath)) { - throw new AutoIngestAlertFileException(String.format("Error creating automated ingest alert file in %s", caseDirectoryPath), ex); - } - } - } - - /** - * Exception thrown when there is a problem creating an alert file. - */ - final static class AutoIngestAlertFileException extends Exception { - - private static final long serialVersionUID = 1L; - - /** - * Constructs an exception to throw when there is a problem creating an - * alert file. - * - * @param message The exception message. - */ - private AutoIngestAlertFileException(String message) { - super(message); - } - - /** - * Constructs an exception to throw when there is a problem creating an - * alert file. - * - * @param message The exception message. - * @param cause The cause of the exception, if it was an exception. - */ - private AutoIngestAlertFileException(String message, Throwable cause) { - super(message, cause); - } - } - - /** - * Prevents instantiation of this utility class. - */ - private AutoIngestAlertFile() { - } - -} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index b621cb514f..a0168cd9ee 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -64,6 +64,7 @@ import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.CaseActionException; import org.sleuthkit.autopsy.casemodule.CaseDetails; import org.sleuthkit.autopsy.casemodule.CaseMetadata; +import org.sleuthkit.autopsy.casemodule.CaseNodeData; import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.Lock; @@ -78,7 +79,6 @@ import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.events.AutopsyEventException; import org.sleuthkit.autopsy.events.AutopsyEventPublisher; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestAlertFile.AutoIngestAlertFileException; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJobLogger.AutoIngestJobLoggerException; import org.sleuthkit.autopsy.experimental.autoingest.FileExporter.FileExportException; import org.sleuthkit.autopsy.experimental.autoingest.ManifestFileParser.ManifestFileParserException; @@ -98,6 +98,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; +import org.sleuthkit.datamodel.Content; /** * An auto ingest manager is responsible for processing auto ingest jobs defined @@ -554,7 +555,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang ++maxPriority; for (AutoIngestJob job : prioritizedJobs) { try { - this.updateCoordinationServiceNode(job); + this.updateCoordinationServiceManifestNode(job); job.setPriority(maxPriority); } catch (CoordinationServiceException | InterruptedException ex) { throw new AutoIngestManagerException("Error updating case priority", ex); @@ -602,13 +603,13 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /* - * Bump the priority by one and update the coordination service node - * data for the job. + * Bump the priority by one and update the coordination service + * manifest node data for the job. */ if (null != prioritizedJob) { ++maxPriority; try { - this.updateCoordinationServiceNode(prioritizedJob); + this.updateCoordinationServiceManifestNode(prioritizedJob); } catch (CoordinationServiceException | InterruptedException ex) { throw new AutoIngestManagerException("Error updating job priority", ex); } @@ -649,7 +650,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang /* * Add the job to the pending jobs queue and update the coordination - * service node data for the job. + * service manifest node data for the job. */ if (null != completedJob && !completedJob.getCaseDirectoryPath().toString().isEmpty()) { try { @@ -661,7 +662,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang completedJob.setCompletedDate(new Date(0)); completedJob.setProcessingStatus(PENDING); completedJob.setProcessingStage(AutoIngestJob.Stage.PENDING, Date.from(Instant.now())); - updateCoordinationServiceNode(completedJob); + updateCoordinationServiceManifestNode(completedJob); pendingJobs.add(completedJob); } catch (CoordinationServiceException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Coordination service error while reprocessing %s", manifestPath), ex); @@ -755,7 +756,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString())); AutoIngestJob deletedJob = new AutoIngestJob(nodeData); deletedJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.DELETED); - this.updateCoordinationServiceNode(deletedJob); + this.updateCoordinationServiceManifestNode(deletedJob); } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); return CaseDeletionResult.PARTIALLY_DELETED; @@ -865,20 +866,31 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /** - * Sets the coordination service node data for an auto ingest job. + * Sets the coordination service manifest node. * - * Note that a new auto ingest node data object will be created from the job - * passed in. Thus, if the data version of the node has changed, the node - * will be "upgraded" as well as updated. + * Note that a new auto ingest job node data object will be created from + * the job passed in. Thus, if the data version of the node has changed, + * the node will be "upgraded" as well as updated. * * @param job The auto ingest job. */ - void updateCoordinationServiceNode(AutoIngestJob job) throws CoordinationServiceException, InterruptedException { + void updateCoordinationServiceManifestNode(AutoIngestJob job) throws CoordinationServiceException, InterruptedException { AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(job); String manifestNodePath = job.getManifest().getFilePath().toString(); byte[] rawData = nodeData.toArray(); coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestNodePath, rawData); } + + /** + * Sets the coordination service case node. + * + * @param caseNodeData The case node data. + * @param caseDirectoryPath The case directory. + */ + void updateCoordinationServiceCaseNode(CaseNodeData caseNodeData, Path caseDirectoryPath) throws CoordinationServiceException, InterruptedException { + byte[] rawData = caseNodeData.toArray(); + coordinationService.setNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString(), rawData); + } /** * A task that submits an input directory scan task to the input directory @@ -1147,8 +1159,8 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /* - * Try to upgrade/update the coordination service node data for - * the job. + * Try to upgrade/update the coordination service manifest node + * data for the job. * * An exclusive lock is obtained before doing so because another * host may have already found the job, obtained an exclusive @@ -1161,7 +1173,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { if (null != manifestLock) { - updateCoordinationServiceNode(job); + updateCoordinationServiceManifestNode(job); } } catch (CoordinationServiceException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); @@ -1186,9 +1198,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ private void addNewPendingJob(Manifest manifest) throws InterruptedException, AutoIngestJobException { /* - * Create the coordination service node data for the job. Note that - * getting the lock will create the node for the job (with no data) - * if it does not already exist. + * Create the coordination service manifest node data for the job. + * Note that getting the lock will create the node for the job + * (with no data) if it does not already exist. * * An exclusive lock is obtained before creating the node data * because another host may have already found the job, obtained an @@ -1202,7 +1214,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { if (null != manifestLock) { AutoIngestJob job = new AutoIngestJob(manifest); - updateCoordinationServiceNode(job); + updateCoordinationServiceManifestNode(job); newPendingJobsList.add(job); } } catch (CoordinationServiceException ex) { @@ -1219,14 +1231,14 @@ public final class AutoIngestManager extends Observable implements PropertyChang * status was not updated. * * @param manifest The manifest for upgrading the node. - * @param nodeData The node data. + * @param jobNodeData The auto ingest job node data. * * @throws InterruptedException if the thread running the input * directory scan task is interrupted while * blocked, i.e., if auto ingest is * shutting down. */ - private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException { + private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData jobNodeData) throws InterruptedException, AutoIngestJobException { /* * Try to get an exclusive lock on the coordination service node for * the job. If the lock cannot be obtained, another host in the auto @@ -1240,7 +1252,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang /* * Create the recovery job. */ - AutoIngestJob job = new AutoIngestJob(nodeData); + AutoIngestJob job = new AutoIngestJob(jobNodeData); int numberOfCrashes = job.getNumberOfCrashes(); ++numberOfCrashes; job.setNumberOfCrashes(numberOfCrashes); @@ -1254,15 +1266,16 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /* - * Update the coordination service node for the job. If - * this fails, leave the recovery to another host. + * Update the coordination service manifest node for + * the job. If this fails, leave the recovery to + * another host. */ try { - updateCoordinationServiceNode(job); + updateCoordinationServiceManifestNode(job); if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { newPendingJobsList.add(job); } else { - newCompletedJobsList.add(new AutoIngestJob(nodeData)); + newCompletedJobsList.add(new AutoIngestJob(jobNodeData)); } } catch (CoordinationServiceException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifestPath), ex); @@ -1270,13 +1283,15 @@ public final class AutoIngestManager extends Observable implements PropertyChang } /* - * Write the alert file and do the logging. + * Update the case node data and do the logging. */ if (null != caseDirectoryPath) { try { - AutoIngestAlertFile.create(nodeData.getCaseDirectoryPath()); - } catch (AutoIngestAlertFileException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error creating alert file for crashed job for %s", manifestPath), ex); + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); + } catch (CaseNodeData.InvalidDataException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to get case node data for %s", caseDirectoryPath), ex); } } if (numberOfCrashes <= AutoIngestUserPreferences.getMaxNumTimesToProcessImage()) { @@ -1292,7 +1307,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); if (null != caseDirectoryPath) { try { - new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), nodeData.getCaseDirectoryPath()).logCrashRecoveryNoRetry(); + new AutoIngestJobLogger(manifest.getFilePath(), manifest.getDataSourceFileName(), jobNodeData.getCaseDirectoryPath()).logCrashRecoveryNoRetry(); } catch (AutoIngestJobLoggerException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error creating case auto ingest log entry for crashed job for %s", manifestPath), ex); } @@ -1352,15 +1367,16 @@ public final class AutoIngestManager extends Observable implements PropertyChang job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); /* - * Try to upgrade/update the coordination service node data - * for the job. It is possible that two hosts will both try - * to obtain the lock to do the upgrade operation at the - * same time. If this happens, the host that is holding the - * lock will complete the upgrade operation. + * Try to upgrade/update the coordination service manifest + * node data for the job. It is possible that two hosts + * will both try to obtain the lock to do the upgrade + * operation at the same time. If this happens, the host + * that is holding the lock will complete the upgrade + * operation. */ try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { if (null != manifestLock) { - updateCoordinationServiceNode(job); + updateCoordinationServiceManifestNode(job); } } catch (CoordinationServiceException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); @@ -1507,8 +1523,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang errorState = ErrorState.ANALYSIS_STARTUP_ERROR; } else if (ex instanceof FileExportException) { errorState = ErrorState.FILE_EXPORT_ERROR; - } else if (ex instanceof AutoIngestAlertFileException) { - errorState = ErrorState.ALERT_FILE_ERROR; } else if (ex instanceof AutoIngestJobLoggerException) { errorState = ErrorState.JOB_LOGGER_ERROR; } else if (ex instanceof AutoIngestDataSourceProcessorException) { @@ -1691,9 +1705,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * @throws FileExportException if there is an * error exporting * files. - * @throws AutoIngestAlertFileException if there is an - * error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an * error writing to * the auto ingest @@ -1710,7 +1721,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * auto ingest node * data objects. */ - private void processJobs() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, AutoIngestJobNodeData.InvalidDataException { + private void processJobs() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, AutoIngestJobNodeData.InvalidDataException, CaseNodeData.InvalidDataException { SYS_LOGGER.log(Level.INFO, "Started processing pending jobs queue"); Lock manifestLock = JobProcessingTask.this.dequeueAndLockNextJob(); while (null != manifestLock) { @@ -1890,9 +1901,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * @throws FileExportException if there is an * error exporting * files. - * @throws AutoIngestAlertFileException if there is an - * error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an * error writing to * the auto ingest @@ -1909,13 +1917,13 @@ public final class AutoIngestManager extends Observable implements PropertyChang * auto ingest node * data objects. */ - private void processJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, AutoIngestJobNodeData.InvalidDataException { + private void processJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException { Path manifestPath = currentJob.getManifest().getFilePath(); SYS_LOGGER.log(Level.INFO, "Started processing of {0}", manifestPath); currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PROCESSING); currentJob.setProcessingStage(AutoIngestJob.Stage.STARTING, Date.from(Instant.now())); currentJob.setProcessingHostName(AutoIngestManager.LOCAL_HOST_NAME); - updateCoordinationServiceNode(currentJob); + updateCoordinationServiceManifestNode(currentJob); setChanged(); notifyObservers(Event.JOB_STARTED); eventPublisher.publishRemotely(new AutoIngestJobStartedEvent(currentJob)); @@ -1939,14 +1947,16 @@ public final class AutoIngestManager extends Observable implements PropertyChang currentJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.PENDING); } currentJob.setProcessingHostName(""); - updateCoordinationServiceNode(currentJob); + updateCoordinationServiceManifestNode(currentJob); boolean retry = (!currentJob.isCanceled() && !currentJob.isCompleted()); SYS_LOGGER.log(Level.INFO, "Completed processing of {0}, retry = {1}", new Object[]{manifestPath, retry}); if (currentJob.isCanceled()) { Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); if (null != caseDirectoryPath) { - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, currentJob.getManifest().getDataSourceFileName(), caseDirectoryPath); jobLogger.logJobCancelled(); } @@ -1994,7 +2004,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * i.e., if auto ingest is * shutting down. */ - private void attemptJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { + private void attemptJob() throws CoordinationServiceException, SharedConfigurationException, ServicesMonitorException, DatabaseServerDownException, KeywordSearchServerDownException, CaseManagementException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException { updateConfiguration(); if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) { return; @@ -2159,8 +2169,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * level ingest modules. * @throws FileExportException if there is an error exporting * files. - * @throws AutoIngestAlertFileException if there is an error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an error writing to * the auto ingest log for the * case. @@ -2169,7 +2177,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * while blocked, i.e., if auto * ingest is shutting down. */ - private void runIngestForJob(Case caseForJob) throws CoordinationServiceException, AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { + private void runIngestForJob(Case caseForJob) throws CoordinationServiceException, AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException { try { if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) { return; @@ -2197,8 +2205,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * level ingest modules. * @throws FileExportException if there is an error exporting * files. - * @throws AutoIngestAlertFileException if there is an error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an error writing to * the auto ingest log for the * case. @@ -2207,7 +2213,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * while blocked, i.e., if auto * ingest is shutting down. */ - private void ingestDataSource(Case caseForJob) throws AnalysisStartupException, FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { + private void ingestDataSource(Case caseForJob) throws AnalysisStartupException, FileExportException, AutoIngestJobLoggerException, InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException, CoordinationServiceException { if (currentJob.isCanceled() || jobProcessingTaskFuture.isCancelled()) { return; } @@ -2255,8 +2261,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * * @return A data source object. * - * @throws AutoIngestAlertFileException if there is an error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an error writing to * the auto ingest log for the * case. @@ -2265,7 +2269,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * interrupted while blocked, i.e., * if auto ingest is shutting down. */ - private DataSource identifyDataSource() throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private DataSource identifyDataSource() throws AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Identifying data source for {0} ", manifestPath); @@ -2277,22 +2281,22 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (!dataSource.exists()) { SYS_LOGGER.log(Level.SEVERE, "Missing data source for {0}", manifestPath); currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logMissingDataSource(); return null; } String deviceId = manifest.getDeviceId(); return new DataSource(deviceId, dataSourcePath); } - + /** * Passes the data source for the current job through a data source * processor that adds it to the case database. * * @param dataSource The data source. * - * @throws AutoIngestAlertFileException if there is an error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an error writing to * the auto ingest log for the * case. @@ -2301,7 +2305,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * while blocked, i.e., if auto * ingest is shutting down. */ - private void runDataSourceProcessor(Case caseForJob, DataSource dataSource) throws InterruptedException, AutoIngestAlertFileException, AutoIngestJobLoggerException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { + private void runDataSourceProcessor(Case caseForJob, DataSource dataSource) throws InterruptedException, AutoIngestJobLoggerException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException, CaseNodeData.InvalidDataException, CoordinationServiceException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Adding data source for {0} ", manifestPath); @@ -2327,7 +2331,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang // did we find a data source processor that can process the data source if (validDataSourceProcessorsMap.isEmpty()) { // This should never happen. We should add all unsupported data sources as logical files. - AutoIngestAlertFile.create(caseDirectoryPath); + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); currentJob.setErrorsOccurred(true); jobLogger.logFailedToIdentifyDataSource(); SYS_LOGGER.log(Level.WARNING, "Unsupported data source {0} for {1}", new Object[]{dataSource.getPath(), manifestPath}); // NON-NLS @@ -2353,7 +2359,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang // Log that the current DSP failed and set the error flag. We consider it an error // if a DSP fails even if a later one succeeds since we expected to be able to process // the data source which each DSP on the list. - AutoIngestAlertFile.create(caseDirectoryPath); + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); currentJob.setErrorsOccurred(true); jobLogger.logDataSourceProcessorError(selectedProcessor.getDataSourceType()); SYS_LOGGER.log(Level.SEVERE, "Exception while processing {0} with data source processor {1}", new Object[]{dataSource.getPath(), selectedProcessor.getDataSourceType()}); @@ -2377,8 +2385,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * * @param dataSource The data source. * - * @throws AutoIngestAlertFileException if there is an error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an error writing to * the auto ingest log for the * case. @@ -2387,7 +2393,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * while blocked, i.e., if auto * ingest is shutting down. */ - private void logDataSourceProcessorResult(DataSource dataSource) throws AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private void logDataSourceProcessorResult(DataSource dataSource) throws AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); @@ -2399,7 +2405,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang jobLogger.logDataSourceAdded(); if (dataSource.getContent().isEmpty()) { currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logNoDataSourceContent(); } break; @@ -2411,7 +2419,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang jobLogger.logDataSourceAdded(); if (dataSource.getContent().isEmpty()) { currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logNoDataSourceContent(); } break; @@ -2421,7 +2431,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.SEVERE, "Critical error running data source processor for {0}: {1}", new Object[]{manifestPath, errorMessage}); } currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logFailedToAddDataSource(); break; } @@ -2435,7 +2447,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang */ SYS_LOGGER.log(Level.WARNING, "Cancellation while waiting for data source processor for {0}", manifestPath); currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logDataSourceProcessorCancelled(); } } @@ -2449,8 +2463,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * * @throws AnalysisStartupException if there is an error analyzing * the data source. - * @throws AutoIngestAlertFileException if there is an error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an error writing to * the auto ingest log for the * case. @@ -2459,7 +2471,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * while blocked, i.e., if auto * ingest is shutting down. */ - private void analyze(DataSource dataSource) throws AnalysisStartupException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private void analyze(DataSource dataSource) throws AnalysisStartupException, AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Starting ingest modules analysis for {0} ", manifestPath); @@ -2491,7 +2503,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (!cancelledModules.isEmpty()) { SYS_LOGGER.log(Level.WARNING, String.format("Ingest module(s) cancelled for %s", manifestPath)); currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); for (String module : snapshot.getCancelledDataSourceIngestModules()) { SYS_LOGGER.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); jobLogger.logIngestModuleCancelled(module); @@ -2501,7 +2515,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang } else { currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logAnalysisCancelled(); CancellationReason cancellationReason = snapshot.getCancellationReason(); if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { @@ -2514,13 +2530,17 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); } currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logIngestModuleStartupErrors(); throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); } else { SYS_LOGGER.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logAnalysisStartupError(); throw new AnalysisStartupException("Ingest manager error starting job", ingestJobStartResult.getStartupException()); } @@ -2529,7 +2549,9 @@ public final class AutoIngestManager extends Observable implements PropertyChang SYS_LOGGER.log(Level.SEVERE, "Ingest job settings error for {0}: {1}", new Object[]{manifestPath, warning}); } currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logIngestJobSettingsErrors(); throw new AnalysisStartupException("Error(s) in ingest job settings"); } @@ -2548,8 +2570,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang * * @throws FileExportException if there is an error exporting * the files. - * @throws AutoIngestAlertFileException if there is an error creating an - * alert file. * @throws AutoIngestJobLoggerException if there is an error writing to * the auto ingest log for the * case. @@ -2558,7 +2578,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang * while blocked, i.e., if auto * ingest is shutting down. */ - private void exportFiles(DataSource dataSource) throws FileExportException, AutoIngestAlertFileException, AutoIngestJobLoggerException, InterruptedException { + private void exportFiles(DataSource dataSource) throws FileExportException, AutoIngestJobLoggerException, InterruptedException, CaseNodeData.InvalidDataException, CoordinationServiceException { Manifest manifest = currentJob.getManifest(); Path manifestPath = manifest.getFilePath(); SYS_LOGGER.log(Level.INFO, "Exporting files for {0}", manifestPath); @@ -2574,11 +2594,13 @@ public final class AutoIngestManager extends Observable implements PropertyChang } catch (FileExportException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex); currentJob.setErrorsOccurred(true); - AutoIngestAlertFile.create(caseDirectoryPath); // Do this first, it is more important than the case log + CaseNodeData caseNodeData = new CaseNodeData(coordinationService.getNodeData(CoordinationService.CategoryNode.CASES, caseDirectoryPath.toString())); + caseNodeData.setErrorsOccurred(true); + updateCoordinationServiceCaseNode(caseNodeData, caseDirectoryPath); jobLogger.logFileExportError(); } } - + /** * A data source processor progress monitor does nothing. There is * currently no mechanism for showing or recording data source processor @@ -2740,7 +2762,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang currentJob.getProcessingStageDetails(); setChanged(); notifyObservers(Event.JOB_STATUS_UPDATED); - updateCoordinationServiceNode(currentJob); + updateCoordinationServiceManifestNode(currentJob); eventPublisher.publishRemotely(new AutoIngestJobStatusEvent(currentJob)); } @@ -2843,7 +2865,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang CASE_MANAGEMENT_ERROR("Case management error"), ANALYSIS_STARTUP_ERROR("Analysis startup error"), FILE_EXPORT_ERROR("File export error"), - ALERT_FILE_ERROR("Alert file error"), JOB_LOGGER_ERROR("Job logger error"), DATA_SOURCE_PROCESSOR_ERROR("Data source processor error"), UNEXPECTED_EXCEPTION("Unknown error"); @@ -2918,7 +2939,6 @@ public final class AutoIngestManager extends Observable implements PropertyChang PARTIALLY_DELETED, FULLY_DELETED } - static final class AutoIngestManagerException extends Exception { private static final long serialVersionUID = 1L;