From 0aa5557b3403ee842b53bd692cbbc83e722f55ec Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 27 Sep 2017 14:51:07 -0400 Subject: [PATCH] Add node data version checks to AIM --- .../autoingest/AutoIngestJobNodeData.java | 28 ++-- .../autoingest/AutoIngestManager.java | 138 ++++++++++-------- 2 files changed, 96 insertions(+), 70 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobNodeData.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobNodeData.java index 4dbedf644e..e2b267fded 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobNodeData.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobNodeData.java @@ -33,13 +33,13 @@ final class AutoIngestJobNodeData { private static final int CURRENT_VERSION = 1; private static final int DEFAULT_PRIORITY = 0; - + /* * This number is the sum of each piece of data, based on it's type. For the * types boolean, int, and long, values 1, 4, and 8 will be added * respectively. For String objects, the length of the string, plus either a * byte or short respesenting the length of the string, will be added. - * + * * This field is used to set the size of the buffer during the byte array * creation in the 'toArray()' method. Since the final size of the array * isn't immediately known at the time of creation, this number is used to @@ -74,6 +74,16 @@ final class AutoIngestJobNodeData { private String processingStageDetailsDescription; // 'byte' length used in byte array private long processingStageDetailsStartDate; + /** + * Gets the current version of the auto ingest job coordination service node + * data. + * + * @return The version number. + */ + static int getCurrentVersion() { + return AutoIngestJobNodeData.CURRENT_VERSION; + } + /** * Uses an auto ingest job to construct an object that converts auto ingest * job data for an auto ingest job coordination service node to and from @@ -333,7 +343,7 @@ final class AutoIngestJobNodeData { * Gets the path to the case directory of the case associated with the job. * * @return The case directory path or an empty string path if the case - * directory has not been created yet. + * directory has not been created yet. */ synchronized Path getCaseDirectoryPath() { if (!caseDirectoryPath.isEmpty()) { @@ -535,10 +545,10 @@ final class AutoIngestJobNodeData { * This method retrieves a string from a given buffer. Depending on the type * specified, either a 'byte' or a 'short' will first be read out of the * buffer which gives the length of the string so it can be properly parsed. - * - * @param buffer The buffer from which the string will be read. + * + * @param buffer The buffer from which the string will be read. * @param lengthType The size of the length data. - * + * * @return The string read from the buffer. */ private String getStringFromBuffer(ByteBuffer buffer, TypeKind lengthType) { @@ -568,10 +578,10 @@ final class AutoIngestJobNodeData { * type specified, either a 'byte' or a 'short' will be inserted prior to * the string which gives the length of the string so it can be properly * parsed. - * + * * @param stringValue The string to write to the buffer. - * @param buffer The buffer to which the string will be written. - * @param lengthType The size of the length data. + * @param buffer The buffer to which the string will be written. + * @param lengthType The size of the length data. */ private void putStringIntoBuffer(String stringValue, ByteBuffer buffer, TypeKind lengthType) { switch (lengthType) { diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 8b519891f1..18d33b5c69 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -344,7 +344,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (event.shouldRetry() == false) { synchronized (jobsLock) { AutoIngestJob job = event.getJob(); - if(completedJobs.contains(job)) { + if (completedJobs.contains(job)) { completedJobs.remove(job); } completedJobs.add(event.getJob()); @@ -1123,33 +1123,47 @@ public final class AutoIngestManager extends Observable implements PropertyChang * shutting down. */ private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException { - AutoIngestJob job = new AutoIngestJob(manifest); - job.setPriority(nodeData.getPriority()); + AutoIngestJob job; + if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) { + job = new AutoIngestJob(nodeData); + Path caseDirectory = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); + if (null != caseDirectory) { + job.setCaseDirectoryPath(caseDirectory); + } + } else { + job = new AutoIngestJob(manifest); + job.setPriority(nodeData.getPriority()); // Retain priority, present in all versions of the node data. + Path caseDirectory = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); + if (null != caseDirectory) { + job.setCaseDirectoryPath(caseDirectory); + } + + /* + * Try to upgrade/update the coordination service node data for + * the job. + * + * An exclusive lock is obtained before doing so because another + * host may have already found the job, obtained an exclusive + * lock, and started processing it. However, this locking does + * make it possible that two processing hosts will both try to + * obtain the lock to do the upgrade operation at the same time. + * If this happens, the host that is holding the lock will + * complete the upgrade operation, so there is nothing more for + * this host to do. + */ + try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { + if (null != manifestLock) { + updateCoordinationServiceNode(job); + } + } catch (CoordinationServiceException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); + } + } Path caseDirectory = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); if (null != caseDirectory) { job.setCaseDirectoryPath(caseDirectory); } newPendingJobsList.add(job); - - /* - * Try to upgrade/update the coordination service node data for the - * job. - * - * An exclusive lock is obtained before doing so because another - * host may have already found the job, obtained an exclusive lock, - * and started processing it. However, this locking does make it - * possible that two hosts will both try to obtain the lock to do - * the upgrade/update operation at the same time. If this happens, - * the host that is holding the lock will complete the - * update/upgrade operation, so there is nothing more to do. - */ - try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { - if (null != manifestLock) { - AutoIngestManager.this.updateCoordinationServiceNode(job); - } - } catch (CoordinationServiceException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); - } } /** @@ -1180,7 +1194,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { if (null != manifestLock) { AutoIngestJob job = new AutoIngestJob(manifest); - AutoIngestManager.this.updateCoordinationServiceNode(job); + updateCoordinationServiceNode(job); newPendingJobsList.add(job); } } catch (CoordinationServiceException ex) { @@ -1303,46 +1317,48 @@ public final class AutoIngestManager extends Observable implements PropertyChang private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException { Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); if (null != caseDirectoryPath) { - /** - * We use the manifest rather than the nodeData here to create - * a new AutoIngestJob instance because the AutoIngestJob - * constructor that takes a nodeData expects the nodeData to - * have fields that do not exist in earlier versions. - */ - AutoIngestJob job = new AutoIngestJob(manifest); - /** - * Update the job with the fields that exist in all versions - * of the nodeData. - */ - job.setCompletedDate(nodeData.getCompletedDate()); - job.setErrorsOccurred(nodeData.getErrorsOccurred()); - job.setPriority(nodeData.getPriority()); - job.setNumberOfCrashes(nodeData.getNumberOfCrashes()); - job.setProcessingStage(AutoIngestJob.Stage.COMPLETED, nodeData.getCompletedDate()); + AutoIngestJob job; + if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) { + job = new AutoIngestJob(nodeData); + job.setCaseDirectoryPath(caseDirectoryPath); + } else { + /** + * Use the manifest rather than the node data here to create + * a new AutoIngestJob instance because the AutoIngestJob + * constructor that takes a node data object expects the + * node data to have fields that do not exist in earlier + * versions. + */ + job = new AutoIngestJob(manifest); + job.setCaseDirectoryPath(caseDirectoryPath); - job.setCaseDirectoryPath(caseDirectoryPath); - job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); - newCompletedJobsList.add(job); + /** + * Update the job with the fields that exist in all versions + * of the nodeData. + */ + job.setCompletedDate(nodeData.getCompletedDate()); + job.setErrorsOccurred(nodeData.getErrorsOccurred()); + job.setPriority(nodeData.getPriority()); + job.setNumberOfCrashes(nodeData.getNumberOfCrashes()); + job.setProcessingStage(AutoIngestJob.Stage.COMPLETED, nodeData.getCompletedDate()); + job.setProcessingStatus(AutoIngestJob.ProcessingStatus.COMPLETED); - /* - * Try to upgrade/update the coordination service node data for - * the job. - * - * An exclusive lock is obtained before doing so because another - * host may have already found the job, obtained an exclusive - * lock, and started processing it. However, this locking does - * make it possible that two hosts will both try to obtain the - * lock to do the upgrade/update operation at the same time. If - * this happens, the host that is holding the lock will complete - * the update/upgrade operation, so there is nothing more to do. - */ - try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { - if (null != manifestLock) { - updateCoordinationServiceNode(job); + /* + * Try to upgrade/update the coordination service node data + * for the job. It is possible that two hosts will both try + * to obtain the lock to do the upgrade operation at the + * same time. If this happens, the host that is holding the + * lock will complete the upgrade operation. + */ + try (Lock manifestLock = coordinationService.tryGetExclusiveLock(CoordinationService.CategoryNode.MANIFESTS, manifest.getFilePath().toString())) { + if (null != manifestLock) { + updateCoordinationServiceNode(job); + } + } catch (CoordinationServiceException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); } - } catch (CoordinationServiceException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to set node data for %s", manifest.getFilePath()), ex); } + newCompletedJobsList.add(job); } else { SYS_LOGGER.log(Level.WARNING, String.format("Job completed for %s, but cannot find case directory, ignoring job", nodeData.getManifestFilePath())); @@ -2553,7 +2569,7 @@ public final class AutoIngestManager extends Observable implements PropertyChang if (fileExporter.isEnabled()) { fileExporter.process(manifest.getDeviceId(), dataSource.getContent(), currentJob::isCanceled); jobLogger.logFileExportCompleted(); - } + } } catch (FileExportException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error doing file export for %s", manifestPath), ex); currentJob.setErrorsOccurred(true);