mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

First cut at cleanup

Browse Source
This commit is contained in:
Eugene Livis 2022-04-12 16:51:21 -04:00
parent bb0de55fae
commit 8774bdb0df
8 changed files with 166 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Experimental/nbproject/project.xml
View File

@ -168,6 +168,7 @@
</module-dependencies>
<public-packages>
<package>org.sleuthkit.autopsy.experimental.autoingest</package>
<package>org.sleuthkit.autopsy.experimental.cleanup</package>
<package>org.sleuthkit.autopsy.experimental.configuration</package>
</public-packages>
<class-path-extension>

121
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

@ -112,6 +112,7 @@ import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchJobSettings;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
/**
* An auto ingest manager is responsible for processing auto ingest jobs defined
@ -185,7 +186,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
private static final String CLEANUP_SCHEDULER_THREAD_NAME = "AIM-cleanup-scheduler-%d";
private final ScheduledThreadPoolExecutor cleanupSchedulingExecutor;
private final ExecutorService cleanupExecutor;
private static final long CLEANUP_INTERVAL_HOURS = 3;
private static final long CLEANUP_INTERVAL_HOURS = 15; // ELTODO
private static final String CLEANUP_THREAD_NAME = "AIM-cleanup-%d";
private volatile AutoIngestNodeStateEvent lastPublishedStateEvent;
@ -270,8 +271,11 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
inputScanSchedulingExecutor.scheduleWithFixedDelay(new InputDirScanSchedulingTask(), 0, AutoIngestUserPreferences.getMinutesOfInputScanInterval(), TimeUnit.MINUTES);
jobProcessingTask = new JobProcessingTask();
jobProcessingTaskFuture = jobProcessingExecutor.submit(jobProcessingTask);
// ELTODO SWITCH TO HOURS
jobStatusPublishingExecutor.scheduleWithFixedDelay(new PeriodicJobStatusEventTask(), JOB_STATUS_EVENT_INTERVAL_SECONDS, JOB_STATUS_EVENT_INTERVAL_SECONDS, TimeUnit.SECONDS);
cleanupSchedulingExecutor.scheduleWithFixedDelay(new CleanupSchedulingTask(), 0, AutoIngestUserPreferences.getHoursOfCleanupInterval(), TimeUnit.HOURS);
cleanupSchedulingExecutor.scheduleWithFixedDelay(new CleanupSchedulingTask(), 0, AutoIngestUserPreferences.getHoursOfCleanupInterval(), TimeUnit.MINUTES);
eventPublisher.addSubscriber(EVENT_LIST, instance);
state = State.RUNNING;
@ -1964,6 +1968,9 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
// force garbage collection to release file handles
System.gc();
// perform optional input and output directory cleanup
cleanup();
}
if (jobProcessingTaskFuture.isCancelled()) {
return;
@ -1975,6 +1982,25 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
manifestLock = JobProcessingTask.this.dequeueAndLockNextJob();
}
}
private void cleanup() {
try {
//discover the registered implementations of automated cleanup
Collection<? extends AutoIngestCleanup> cleanups
= Lookup.getDefault().lookupAll(AutoIngestCleanup.class);
if (!cleanups.isEmpty()) {
AutoIngestCleanup cleanup = cleanups.iterator().next();
// ELTODO cleanup.runCleanupTask();
}
// loop over all completed jobs and delete input and output for that job
} catch (Exception ex) {
sysLogger.log(Level.SEVERE, "Unexpected exception in CleanupSchedulingTask", ex); //NON-NLS
}
}
/**
* Inspects the pending jobs queue, looking for the next job that is
@ -3140,14 +3166,101 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
if (!cleanups.isEmpty()) {
AutoIngestCleanup cleanup = cleanups.iterator().next();
cleanup.runCleanupTask();
}
sysLogger.log(Level.INFO, "CleanupSchedulingTask - trying to get ingest job lock");
synchronized (jobsLock) {
sysLogger.log(Level.INFO, "CleanupSchedulingTask - got ingest job lock");
for (Iterator<AutoIngestJob> iterator = completedJobs.iterator(); iterator.hasNext();) {
AutoIngestJob job = iterator.next();
// do cleanup for each job
Path casePath = job.getCaseDirectoryPath();
Path dsPath = job.getManifest().getDataSourcePath();
sysLogger.log(Level.INFO, "Cleaning up case {0} for job {1}", new Object[]{casePath.toString(), dsPath.toString()});
cleanup.runCleanupTask(casePath, AutoIngestCleanup.DeleteOptions.DELETE_INPUT_AND_OUTPUT, new DoNothingProgressIndicator());
sysLogger.log(Level.INFO, "Cleanup task completed for this job");
// ELTODO remove completed job?
// verify that the data source, manifest, and case directory have indeed been deleted
if (dsPath.toFile().exists()) {
// data source have NOT ben deleted - keep the completed job so that we
// attempt the cleanup again later
sysLogger.log(Level.SEVERE, "Data source has not been deleted: {0}", dsPath.toString());
continue;
}
if (casePath.toFile().exists()) {
// case output directory has NOT ben deleted - keep the completed job so that we
// attempt the cleanup again later
sysLogger.log(Level.SEVERE, "Case directory has not been deleted: {0}", casePath.toString());
continue;
}
// if we are here then everything got deleted - remove the completed job
// ELTODO do we need this, since we are doing input folder scan at the end?
iterator.remove();
// send message that case has been deleted
new Thread(() -> {
eventPublisher.publishRemotely(new AutoIngestCaseDeletedEvent(LOCAL_HOST_NAME, job.getManifest().getCaseName(),
getSystemUserNameProperty()));
}).start();
}
// trigger input scan which will update the ZK nodes and tables
// ELTODO should we do this inside the jobsLock or outside?
scanInputDirsNow();
}
}
} catch (Exception ex) {
sysLogger.log(Level.SEVERE, "Unexpected exception in CleanupSchedulingTask", ex); //NON-NLS
}
}
}
/**
* A data source processor progress monitor does nothing. There is currently
* no mechanism for showing or recording data source processor progress
* during an ingest job.
*/
private class DoNothingProgressIndicator implements ProgressIndicator {
@Override
public void start(String message, int totalWorkUnits) {
}
@Override
public void start(String message) {
}
@Override
public void switchToIndeterminate(String message) {
}
@Override
public void switchToDeterminate(String message, int workUnitsCompleted, int totalWorkUnits) {
}
@Override
public void progress(String message) {
}
@Override
public void progress(int workUnitsCompleted) {
}
@Override
public void progress(String message, int workUnitsCompleted) {
}
@Override
public void finish() {
}
}
/**
* An instance of this runnable is responsible for periodically sending auto

3
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DeleteCaseAction.java
View File

@ -23,6 +23,7 @@ import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.multiusercases.CaseNodeData;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
import org.sleuthkit.autopsy.experimental.cleanup.AutoIngestCleanup.DeleteOptions;
/**
* An action that completely deletes one or more multi-user cases. Only the
@ -69,6 +70,6 @@ final class DeleteCaseAction extends DeleteCaseComponentsAction {
@Override
DeleteCaseTask getTask(CaseNodeData caseNodeData, ProgressIndicator progress) {
return new DeleteCaseTask(caseNodeData, DeleteCaseTask.DeleteOptions.DELETE_CASE, progress);
return new DeleteCaseTask(caseNodeData, DeleteOptions.DELETE_CASE, progress);
}
}

2
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DeleteCaseInputAction.java
View File

@ -22,7 +22,7 @@ import java.awt.event.ActionEvent;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.multiusercases.CaseNodeData;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.experimental.autoingest.DeleteCaseTask.DeleteOptions;
import org.sleuthkit.autopsy.experimental.cleanup.AutoIngestCleanup.DeleteOptions;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
/**

2
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DeleteCaseInputAndOutputAction.java
View File

@ -23,7 +23,7 @@ import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.multiusercases.CaseNodeData;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.experimental.autoingest.DeleteCaseTask.DeleteOptions;
import org.sleuthkit.autopsy.experimental.cleanup.AutoIngestCleanup.DeleteOptions;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
/**

2
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DeleteCaseOutputAction.java
View File

@ -22,7 +22,7 @@ import java.awt.event.ActionEvent;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.multiusercases.CaseNodeData;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.experimental.autoingest.DeleteCaseTask.DeleteOptions;
import org.sleuthkit.autopsy.experimental.cleanup.AutoIngestCleanup.DeleteOptions;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
/**

40
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/DeleteCaseTask.java
View File

@ -49,6 +49,7 @@ import org.sleuthkit.autopsy.coreutils.FileUtil;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJobNodeData.InvalidDataException;
import org.sleuthkit.autopsy.experimental.cleanup.AutoIngestCleanup.DeleteOptions;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.TskCoreException;
@ -58,7 +59,7 @@ import org.sleuthkit.datamodel.TskCoreException;
* directed to the dedicated auto ingest dashboard log instead of to the general
* application log.
*/
final class DeleteCaseTask implements Runnable {
public final class DeleteCaseTask implements Runnable {
private static final int MANIFEST_FILE_LOCKING_TIMEOUT_MINS = 5;
private static final int MANIFEST_DELETE_TRIES = 3;
@ -70,41 +71,6 @@ final class DeleteCaseTask implements Runnable {
private CoordinationService coordinationService;
private CaseMetadata caseMetadata;
/**
* Options to support implementing different case deletion use cases.
*/
enum DeleteOptions {
/**
* Delete the auto ingest job manifests and corresponding data sources,
* while leaving the manifest file coordination service nodes and the
* rest of the case intact. The use case is freeing auto ingest input
* directory space while retaining the option to restore the data
* sources, effectively restoring the case.
*/
DELETE_INPUT,
/**
* Delete the manifest file coordination service nodes and the output
* for a case, while leaving the auto ingest job manifests and
* corresponding data sources intact. The use case is auto ingest
* reprocessing of a case with a clean slate without having to restore
* the manifests and data sources.
*/
DELETE_OUTPUT,
/**
* Delete everything.
*/
DELETE_INPUT_AND_OUTPUT,
/**
* Delete only the case components that the application created. This is
* DELETE_OUTPUT with the additional feature that manifest file
* coordination service nodes are marked as deleted, rather than
* actually deleted. This eliminates the requirement that manifests and
* data sources have to be deleted before deleting the case to avoid an
* unwanted, automatic reprocessing of the case.
*/
DELETE_CASE
}
/**
* Constructs a task that deletes part or all of a given case. Note that all
* logging is directed to the dedicated auto ingest dashboard log instead of
@ -115,7 +81,7 @@ final class DeleteCaseTask implements Runnable {
* @param deleteOption The deletion option for the task.
* @param progress A progress indicator.
*/
DeleteCaseTask(CaseNodeData caseNodeData, DeleteOptions deleteOption, ProgressIndicator progress) {
public DeleteCaseTask(CaseNodeData caseNodeData, DeleteOptions deleteOption, ProgressIndicator progress) {
this.caseNodeData = caseNodeData;
this.deleteOption = deleteOption;
this.progress = progress;

41
Experimental/src/org/sleuthkit/autopsy/experimental/cleanup/AutoIngestCleanup.java
View File

@ -18,11 +18,50 @@
*/
package org.sleuthkit.autopsy.experimental.cleanup;
import java.nio.file.Path;
import org.sleuthkit.autopsy.casemodule.multiusercases.CaseNodeData;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
/**
* Interface to perform automated cleanup of auto ingest input and output directories,
* as well as ZK nodes.
*/
public interface AutoIngestCleanup {
void runCleanupTask();
/**
* Options to support implementing different case deletion use cases.
*/
public enum DeleteOptions {
/**
* Delete the auto ingest job manifests and corresponding data sources,
* while leaving the manifest file coordination service nodes and the
* rest of the case intact. The use case is freeing auto ingest input
* directory space while retaining the option to restore the data
* sources, effectively restoring the case.
*/
DELETE_INPUT,
/**
* Delete the manifest file coordination service nodes and the output
* for a case, while leaving the auto ingest job manifests and
* corresponding data sources intact. The use case is auto ingest
* reprocessing of a case with a clean slate without having to restore
* the manifests and data sources.
*/
DELETE_OUTPUT,
/**
* Delete everything.
*/
DELETE_INPUT_AND_OUTPUT,
/**
* Delete only the case components that the application created. This is
* DELETE_OUTPUT with the additional feature that manifest file
* coordination service nodes are marked as deleted, rather than
* actually deleted. This eliminates the requirement that manifests and
* data sources have to be deleted before deleting the case to avoid an
* unwanted, automatic reprocessing of the case.
*/
DELETE_CASE
}
void runCleanupTask(Path caseOutputDirectoryPath, DeleteOptions deleteOption, ProgressIndicator progress);
}