From 8b36eebe70086bcba507f6ef531d82deb72ca3b2 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 18 Dec 2020 15:08:48 -0500 Subject: [PATCH] Interim check in for EFE hang solution --- .../EmbeddedFileExtractorIngestModule.java | 7 +++++ .../FileTaskExecutor.java | 10 +++--- .../autopsy/threadutils/TaskRetryUtil.java | 31 ++++++++++++++++++- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java index c1994d33b9..5883c74f84 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; import java.nio.file.Paths; import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.datamodel.AbstractFile; @@ -29,10 +30,13 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import net.sf.sevenzipjbinding.SevenZipNativeInitializationException; +import org.sleuthkit.autopsy.apputils.ApplicationLoggers; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.autopsy.modules.embeddedfileextractor.SevenZipExtractor.Archive; +import org.sleuthkit.autopsy.threadutils.TaskRetryUtil; /** * A file level ingest module that extracts embedded files from supported @@ -47,6 +51,7 @@ import org.sleuthkit.autopsy.modules.embeddedfileextractor.SevenZipExtractor.Arc }) public final class EmbeddedFileExtractorIngestModule extends FileIngestModuleAdapter { + private static final String TASK_RETRY_STATS_LOG_NAME = "task_retry_stats"; //Outer concurrent hashmap with keys of JobID, inner concurrentHashmap with keys of objectID private static final ConcurrentHashMap> mapOfDepthTrees = new ConcurrentHashMap<>(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); @@ -189,6 +194,8 @@ public final class EmbeddedFileExtractorIngestModule extends FileIngestModuleAda fileTaskExecutor.shutDown(); if (refCounter.decrementAndGet(jobId) == 0) { mapOfDepthTrees.remove(jobId); + Logger logger = ApplicationLoggers.getLogger(TASK_RETRY_STATS_LOG_NAME); + logger.log(Level.INFO, String.format("total task retries: %d, total task timeouts: %d, total task failures: %d", TaskRetryUtil.getTotalTaskRetriesCount(), TaskRetryUtil.getTotalTaskTimeOutsCount(), TaskRetryUtil.getTotalFailedTasksCount())); } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/FileTaskExecutor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/FileTaskExecutor.java index 08580e2e78..0cbe43a1fb 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/FileTaskExecutor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/FileTaskExecutor.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.sleuthkit.autopsy.apputils.ApplicationLoggers; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.threadutils.TaskRetryUtil; @@ -49,13 +50,14 @@ import org.sleuthkit.autopsy.threadutils.TaskRetryUtil; */ class FileTaskExecutor { - private static final Logger logger = Logger.getLogger(FileTaskExecutor.class.getName()); private static final int MIN_THREADS_IN_POOL = 4; private static final int MAX_THREADS_IN_POOL = Integer.MAX_VALUE; private static final String FILE_IO_TASK_THREAD_NAME = "file-io-executor-task-%d"; - private static final String FILE_EXISTS_TASK_DESC_FMT_STR = "Checking if %s already exists"; - private static final String MKDIRS_TASK_DESC_FMT_STR = "Making directory %s"; - private static final String NEW_FILE_TASK_DESC_FMT_STR = "Creating new file %s"; + private static final String FILE_EXISTS_TASK_DESC_FMT_STR = "Checking if %s already exists (case directory = %s)"; + private static final String MKDIRS_TASK_DESC_FMT_STR = "Making directory %s (case directory = %s)"; + private static final String NEW_FILE_TASK_DESC_FMT_STR = "Creating new file %s (case directory = %s)"; + private static final String FILE_OPS_LOG_NAME = "efe_file-ops"; + private static final Logger logger = ApplicationLoggers.getLogger(FILE_OPS_LOG_NAME); private final ScheduledThreadPoolExecutor executor; private final TaskTerminator terminator; diff --git a/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java b/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java index 89645a395e..6c76c60092 100755 --- a/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java +++ b/Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java @@ -218,8 +218,37 @@ public class TaskRetryUtil { return result; } + /** + * RJCTODO + * + * @return + */ + public static long getTotalTaskRetriesCount() { + return totalTaskRetries.get(); + } + + /** + * RJCTODO + * + * @return + */ + public static long getTotalTaskTimeOutsCount() { + return totalTaskTimeOuts.get(); + } + + /** + * RJCTODO + * + * @return + */ + public static long getTotalFailedTasksCount() { + return totalFailedTasks.get(); + } + + /** + * Private contructor to prevent TaskRetryUtil object instantiation. + */ private TaskRetryUtil() { - // Private contructor to prevent TaskRetryUtil object instantiation. } }