From 9957516926103c84228020633132289b8c5b68f7 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 11 Apr 2014 12:15:58 -0400 Subject: [PATCH 1/6] Extract module counter into its own class. --- .../autopsy/ingest/IngestModuleAdapter.java | 22 --------- .../ingest/ModuleReferenceCounter.java | 49 +++++++++++++++++++ 2 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java index d0ac2e3f32..4187704bc1 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java @@ -18,33 +18,11 @@ */ package org.sleuthkit.autopsy.ingest; -import java.util.HashMap; - /** * An adapter that provides a default implementation of the IngestModule * interface. */ public abstract class IngestModuleAdapter implements IngestModule { - // Maps a JobId to the count of instances - static HashMap moduleRefCount = new HashMap<>(); - - public static synchronized long moduleRefCountIncrementAndGet(long jobId) { - long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0; - long nextCount = count + 1; - moduleRefCount.put(jobId, nextCount); - return nextCount; - } - - public static synchronized long moduleRefCountDecrementAndGet(long jobId) { - if (moduleRefCount.containsKey(jobId)) { - long count = moduleRefCount.get(jobId); - moduleRefCount.put(jobId, --count); - return count; - } else { - return 0; - } - } - @Override public void startUp(IngestJobContext context) throws IngestModuleException { } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java b/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java new file mode 100644 index 0000000000..5afeeb0af0 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java @@ -0,0 +1,49 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011 - 2013 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.sleuthkit.autopsy.ingest; + +import java.util.HashMap; + +/** + * A utility class that modules can use to keep track of whether they are the + * first/last instance for a particular job. + */ +public class ModuleReferenceCounter { + // Maps a JobId to the count of instances + private static HashMap moduleRefCount = new HashMap<>(); + + public static synchronized long moduleRefCountIncrementAndGet(long jobId) { + long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0; + long nextCount = count + 1; + moduleRefCount.put(jobId, nextCount); + return nextCount; + } + + public static synchronized long moduleRefCountDecrementAndGet(long jobId) { + if (moduleRefCount.containsKey(jobId)) { + long count = moduleRefCount.get(jobId); + moduleRefCount.put(jobId, --count); + return count; + } else { + return 0; + } + } +} From 350fbe93594779f2dc53776042ceaa5135b69ae6 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 11 Apr 2014 12:40:05 -0400 Subject: [PATCH 2/6] update ModuleReferenceCounter --- .../sleuthkit/autopsy/ingest/ModuleReferenceCounter.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java b/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java index 5afeeb0af0..8f1ba22a76 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java @@ -25,19 +25,21 @@ import java.util.HashMap; /** * A utility class that modules can use to keep track of whether they are the * first/last instance for a particular job. + * + * An instance of this should be static in your module class. */ public class ModuleReferenceCounter { // Maps a JobId to the count of instances - private static HashMap moduleRefCount = new HashMap<>(); + private HashMap moduleRefCount = new HashMap<>(); - public static synchronized long moduleRefCountIncrementAndGet(long jobId) { + public synchronized long moduleRefCountIncrementAndGet(long jobId) { long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0; long nextCount = count + 1; moduleRefCount.put(jobId, nextCount); return nextCount; } - public static synchronized long moduleRefCountDecrementAndGet(long jobId) { + public synchronized long moduleRefCountDecrementAndGet(long jobId) { if (moduleRefCount.containsKey(jobId)) { long count = moduleRefCount.get(jobId); moduleRefCount.put(jobId, --count); From b5f3d651180fba9879561c1a2c4b756e0340efcb Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 11 Apr 2014 13:20:02 -0400 Subject: [PATCH 3/6] adjusted modules to use new class ModuleReferenceCounter --- .../examples/SampleDataSourceIngestModule.java | 14 ++++++++++++-- .../autopsy/examples/SampleFileIngestModule.java | 13 +++++++++++-- .../autopsy/ingest/ModuleReferenceCounter.java | 8 ++++++-- .../modules/exif/ExifParserFileIngestModule.java | 8 +++++--- .../FileExtMismatchIngestModule.java | 6 ++++-- .../modules/filetypeid/FileTypeIdIngestModule.java | 6 ++++-- .../modules/sevenzip/SevenZipIngestModule.java | 4 +++- .../autopsy/hashdatabase/HashDbIngestModule.java | 14 ++++++++------ .../keywordsearch/KeywordSearchIngestModule.java | 8 +++++--- 9 files changed, 58 insertions(+), 23 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java index a5e329beb5..af9af1c2f1 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java @@ -48,6 +48,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; import org.sleuthkit.datamodel.TskData; /** @@ -60,6 +61,7 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo private static final HashMap fileCountsForIngestJobs = new HashMap<>(); private final boolean skipKnownFiles; private IngestJobContext context = null; + private static ModuleReferenceCounter refCounter = null; SampleDataSourceIngestModule(SampleModuleIngestJobSettings settings) { this.skipKnownFiles = settings.skipKnownFiles(); @@ -129,7 +131,7 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo } synchronized static void initFileCount(long ingestJobId) { - Long refCount = IngestModuleAdapter.moduleRefCountIncrementAndGet(ingestJobId); + Long refCount = getRefCounter().incrementAndGet(ingestJobId); if (refCount == 1) { fileCountsForIngestJobs.put(ingestJobId, 0L); } @@ -142,7 +144,7 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo } synchronized static void postFileCount(long ingestJobId) { - Long refCount = IngestModuleAdapter.moduleRefCountDecrementAndGet(ingestJobId); + Long refCount = getRefCounter().decrementAndGet(ingestJobId); if (refCount == 0) { Long filesCount = fileCountsForIngestJobs.remove(ingestJobId); String msgText = String.format("Found %d files", filesCount); @@ -153,4 +155,12 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo IngestServices.getInstance().postMessage(message); } } + + synchronized private static ModuleReferenceCounter getRefCounter() { + if (refCounter == null) { + refCounter = new ModuleReferenceCounter(); + } + return refCounter; + } + } diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java index 95a64ce126..ee0115e183 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java @@ -40,6 +40,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -59,6 +60,7 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo private static int attrId = -1; private final boolean skipKnownFiles; private IngestJobContext context = null; + private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); SampleFileIngestModule(SampleModuleIngestJobSettings settings) { this.skipKnownFiles = settings.skipKnownFiles(); @@ -167,7 +169,7 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo } synchronized static void initBlackboardPostCount(long ingestJobId) { - Long refCount = IngestModuleAdapter.moduleRefCountIncrementAndGet(ingestJobId); + Long refCount = getRefCounter().incrementAndGet(ingestJobId); if (refCount == 1) { artifactCountsForIngestJobs.put(ingestJobId, 0L); } @@ -180,7 +182,7 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo } synchronized static void reportBlackboardPostCount(long ingestJobId) { - Long refCount = IngestModuleAdapter.moduleRefCountDecrementAndGet(ingestJobId); + Long refCount = getRefCounter().decrementAndGet(ingestJobId); if (refCount == 0) { Long filesCount = artifactCountsForIngestJobs.remove(ingestJobId); String msgText = String.format("Posted %d times to the blackboard", filesCount); @@ -191,4 +193,11 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo IngestServices.getInstance().postMessage(message); } } + + synchronized private static ModuleReferenceCounter getRefCounter() { + if (refCounter == null) { + refCounter = new ModuleReferenceCounter(); + } + return refCounter; + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java b/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java index 8f1ba22a76..4ea2997106 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java @@ -32,14 +32,18 @@ public class ModuleReferenceCounter { // Maps a JobId to the count of instances private HashMap moduleRefCount = new HashMap<>(); - public synchronized long moduleRefCountIncrementAndGet(long jobId) { + public synchronized long get(long jobId) { + return moduleRefCount.get(jobId); + } + + public synchronized long incrementAndGet(long jobId) { long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0; long nextCount = count + 1; moduleRefCount.put(jobId, nextCount); return nextCount; } - public synchronized long moduleRefCountDecrementAndGet(long jobId) { + public synchronized long decrementAndGet(long jobId) { if (moduleRefCount.containsKey(jobId)) { long count = moduleRefCount.get(jobId); moduleRefCount.put(jobId, --count); diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 5c55c5e1bd..9dbf3ff299 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -41,6 +41,7 @@ import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -62,14 +63,15 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem private AtomicInteger filesProcessed = new AtomicInteger(0); private volatile boolean filesToFire = false; private long jobId; - + private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + ExifParserFileIngestModule() { } @Override public void startUp(IngestJobContext context) throws IngestModuleException { jobId = context.getJobId(); - IngestModuleAdapter.moduleRefCountIncrementAndGet(jobId); + refCounter.incrementAndGet(jobId); } @@ -198,7 +200,7 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem @Override public void shutDown(boolean ingestJobCancelled) { // We only need to check for this final event on the last module per job - if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + if (refCounter.decrementAndGet(jobId) == 0) { if (filesToFire) { //send the final new data event services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java index 3c3506e3d7..a9934c3e46 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java @@ -33,6 +33,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -53,6 +54,7 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements private long jobId; private static AtomicLong processTime = new AtomicLong(0); private static AtomicLong numFiles = new AtomicLong(0); + private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) { this.settings = settings; @@ -61,7 +63,7 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements @Override public void startUp(IngestJobContext context) throws IngestModuleException { jobId = context.getJobId(); - IngestModuleAdapter.moduleRefCountIncrementAndGet(jobId); + refCounter.incrementAndGet(jobId); FileExtMismatchXML xmlLoader = FileExtMismatchXML.getDefault(); SigTypeToExtMap = xmlLoader.load(); } @@ -153,7 +155,7 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements @Override public void shutDown(boolean ingestJobCancelled) { // We only need to post the summary msg from the last module per job - if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + if (refCounter.decrementAndGet(jobId) == 0) { StringBuilder detailsSb = new StringBuilder(); detailsSb.append(""); detailsSb.append(""); diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index 3bc7dc68d1..b8675c1447 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -35,6 +35,7 @@ import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; /** * Detects the type of a file based on signature (magic) values. Posts results @@ -48,6 +49,7 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI private long jobId; private static AtomicLong matchTime = new AtomicLong(0); private static AtomicLong numFiles = new AtomicLong(0); + private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); // The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed. // If desired in the future to be more knowledgable about weird files or rare formats, we could @@ -61,7 +63,7 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI @Override public void startUp(IngestJobContext context) throws IngestModuleException { jobId = context.getJobId(); - IngestModuleAdapter.moduleRefCountIncrementAndGet(jobId); + refCounter.incrementAndGet(jobId); } @Override @@ -108,7 +110,7 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI @Override public void shutDown(boolean ingestJobCancelled) { // We only need to post the summary msg from the last module per job - if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + if (refCounter.decrementAndGet(jobId) == 0) { StringBuilder detailsSb = new StringBuilder(); detailsSb.append("
").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("
"); detailsSb.append(""); diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index 82134a7e7e..b29c7becdf 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -60,6 +60,7 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; /** * 7Zip ingest module extracts supported archives, adds extracted DerivedFiles, @@ -91,6 +92,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F private static final int ZIP_SIGNATURE_BE = 0x504B0304; private IngestJobContext context; private long jobId; + private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); SevenZipIngestModule() { } @@ -124,7 +126,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F } // if first instance of this module for this job then check 7zip init - if (IngestModuleAdapter.moduleRefCountIncrementAndGet(jobId) == 1) { + if (refCounter.incrementAndGet(jobId) == 1) { try { SevenZip.initSevenZipFromPlatformJAR(); String platform = SevenZip.getUsedPlatform(); diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index f56bac451e..1767714031 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -43,6 +43,7 @@ import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; import org.sleuthkit.datamodel.HashInfo; public class HashDbIngestModule extends IngestModuleAdapter implements FileIngestModule { @@ -56,10 +57,11 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges private List knownBadHashSets = new ArrayList<>(); private List knownHashSets = new ArrayList<>(); private long jobId; - static AtomicLong totalKnownBadCount = new AtomicLong(0); - static AtomicLong totalCalctime = new AtomicLong(0); - static AtomicLong totalLookuptime = new AtomicLong(0); - + private static AtomicLong totalKnownBadCount = new AtomicLong(0); + private static AtomicLong totalCalctime = new AtomicLong(0); + private static AtomicLong totalLookuptime = new AtomicLong(0); + private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + HashDbIngestModule(HashLookupModuleSettings settings) { this.settings = settings; } @@ -70,7 +72,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets); - if (IngestModuleAdapter.moduleRefCountIncrementAndGet(jobId) == 1) { + if (refCounter.incrementAndGet(jobId) == 1) { // if first module for this job then post error msgs if needed if (knownBadHashSets.isEmpty()) { @@ -303,7 +305,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges @Override public void shutDown(boolean ingestJobCancelled) { - if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + if (refCounter.decrementAndGet(jobId) == 0) { if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) { StringBuilder detailsSb = new StringBuilder(); //details diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 392c7c238d..5f4c0d34a7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -38,6 +38,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ReadContentInputStream; @@ -91,7 +92,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private long dataSourceId; private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging private int instanceNum = 0; - + private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private enum IngestStatus { TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested @@ -130,7 +132,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme // increment the module reference count // if first instance of this module for this job then check the server and existence of keywords - if (IngestModuleAdapter.moduleRefCountIncrementAndGet(jobId) == 1) { + if (refCounter.incrementAndGet(jobId) == 1) { final Server server = KeywordSearch.getServer(); try { if (!server.isRunning()) { @@ -256,7 +258,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme SearchRunner.getInstance().endJob(jobId); // We only need to post the summary msg from the last module per job - if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) { + if (refCounter.decrementAndGet(jobId) == 0) { postIndexSummary(); } From 3471538562cc5936d6b2057aa6c2bc7431b930fd Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 11 Apr 2014 13:27:15 -0400 Subject: [PATCH 4/6] made the refCounter declaration lazy and consistent with data source example --- .../org/sleuthkit/autopsy/examples/SampleFileIngestModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java index ee0115e183..769f3a1047 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java @@ -60,7 +60,7 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo private static int attrId = -1; private final boolean skipKnownFiles; private IngestJobContext context = null; - private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private static ModuleReferenceCounter refCounter = null; SampleFileIngestModule(SampleModuleIngestJobSettings settings) { this.skipKnownFiles = settings.skipKnownFiles(); From 77cca50838c89691e1d88cba9a0c9d4302303ddd Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 11 Apr 2014 14:55:30 -0400 Subject: [PATCH 5/6] renamed ModuleReferenceCounter to IngestModuleReferenceCounter as requested, the class now removes entries once they decrement to zero, and removed static creation methods from sample classes --- .../examples/SampleDataSourceIngestModule.java | 15 ++++----------- .../examples/SampleFileIngestModule.java | 17 +++++------------ ...r.java => IngestModuleReferenceCounter.java} | 10 +++++++--- .../exif/ExifParserFileIngestModule.java | 4 ++-- .../FileExtMismatchIngestModule.java | 4 ++-- .../filetypeid/FileTypeIdIngestModule.java | 4 ++-- .../modules/sevenzip/SevenZipIngestModule.java | 4 ++-- .../hashdatabase/HashDbIngestModule.java | 4 ++-- .../KeywordSearchIngestModule.java | 4 ++-- 9 files changed, 28 insertions(+), 38 deletions(-) rename Core/src/org/sleuthkit/autopsy/ingest/{ModuleReferenceCounter.java => IngestModuleReferenceCounter.java} (87%) diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java index af9af1c2f1..79dc70ce11 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleDataSourceIngestModule.java @@ -48,7 +48,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.TskData; /** @@ -61,7 +61,7 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo private static final HashMap fileCountsForIngestJobs = new HashMap<>(); private final boolean skipKnownFiles; private IngestJobContext context = null; - private static ModuleReferenceCounter refCounter = null; + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); SampleDataSourceIngestModule(SampleModuleIngestJobSettings settings) { this.skipKnownFiles = settings.skipKnownFiles(); @@ -131,7 +131,7 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo } synchronized static void initFileCount(long ingestJobId) { - Long refCount = getRefCounter().incrementAndGet(ingestJobId); + Long refCount = refCounter.incrementAndGet(ingestJobId); if (refCount == 1) { fileCountsForIngestJobs.put(ingestJobId, 0L); } @@ -144,7 +144,7 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo } synchronized static void postFileCount(long ingestJobId) { - Long refCount = getRefCounter().decrementAndGet(ingestJobId); + Long refCount = refCounter.decrementAndGet(ingestJobId); if (refCount == 0) { Long filesCount = fileCountsForIngestJobs.remove(ingestJobId); String msgText = String.format("Found %d files", filesCount); @@ -155,12 +155,5 @@ class SampleDataSourceIngestModule extends IngestModuleAdapter implements DataSo IngestServices.getInstance().postMessage(message); } } - - synchronized private static ModuleReferenceCounter getRefCounter() { - if (refCounter == null) { - refCounter = new ModuleReferenceCounter(); - } - return refCounter; - } } diff --git a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java index 769f3a1047..988d834904 100755 --- a/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java @@ -40,7 +40,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -60,7 +60,7 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo private static int attrId = -1; private final boolean skipKnownFiles; private IngestJobContext context = null; - private static ModuleReferenceCounter refCounter = null; + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); SampleFileIngestModule(SampleModuleIngestJobSettings settings) { this.skipKnownFiles = settings.skipKnownFiles(); @@ -169,7 +169,7 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo } synchronized static void initBlackboardPostCount(long ingestJobId) { - Long refCount = getRefCounter().incrementAndGet(ingestJobId); + Long refCount = refCounter.incrementAndGet(ingestJobId); if (refCount == 1) { artifactCountsForIngestJobs.put(ingestJobId, 0L); } @@ -182,7 +182,7 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo } synchronized static void reportBlackboardPostCount(long ingestJobId) { - Long refCount = getRefCounter().decrementAndGet(ingestJobId); + Long refCount = refCounter.decrementAndGet(ingestJobId); if (refCount == 0) { Long filesCount = artifactCountsForIngestJobs.remove(ingestJobId); String msgText = String.format("Posted %d times to the blackboard", filesCount); @@ -192,12 +192,5 @@ class SampleFileIngestModule extends IngestModuleAdapter implements FileIngestMo msgText); IngestServices.getInstance().postMessage(message); } - } - - synchronized private static ModuleReferenceCounter getRefCounter() { - if (refCounter == null) { - refCounter = new ModuleReferenceCounter(); - } - return refCounter; - } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleReferenceCounter.java similarity index 87% rename from Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java rename to Core/src/org/sleuthkit/autopsy/ingest/IngestModuleReferenceCounter.java index 4ea2997106..1382be299c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/ModuleReferenceCounter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleReferenceCounter.java @@ -28,7 +28,7 @@ import java.util.HashMap; * * An instance of this should be static in your module class. */ -public class ModuleReferenceCounter { +public class IngestModuleReferenceCounter { // Maps a JobId to the count of instances private HashMap moduleRefCount = new HashMap<>(); @@ -46,10 +46,14 @@ public class ModuleReferenceCounter { public synchronized long decrementAndGet(long jobId) { if (moduleRefCount.containsKey(jobId)) { long count = moduleRefCount.get(jobId); - moduleRefCount.put(jobId, --count); + if (--count == 0) { + moduleRefCount.remove(jobId); + } else { + moduleRefCount.put(jobId, count); + } return count; } else { - return 0; + return -1; } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java index 9dbf3ff299..6e9f8c4ecf 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java @@ -41,7 +41,7 @@ import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -63,7 +63,7 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem private AtomicInteger filesProcessed = new AtomicInteger(0); private volatile boolean filesToFire = false; private long jobId; - private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); ExifParserFileIngestModule() { } diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java index a9934c3e46..5745017a08 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java @@ -33,7 +33,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -54,7 +54,7 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements private long jobId; private static AtomicLong processTime = new AtomicLong(0); private static AtomicLong numFiles = new AtomicLong(0); - private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) { this.settings = settings; diff --git a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java index b8675c1447..0ba451f716 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java @@ -35,7 +35,7 @@ import org.sleuthkit.datamodel.TskData.FileKnown; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; /** * Detects the type of a file based on signature (magic) values. Posts results @@ -49,7 +49,7 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI private long jobId; private static AtomicLong matchTime = new AtomicLong(0); private static AtomicLong numFiles = new AtomicLong(0); - private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); // The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed. // If desired in the future to be more knowledgable about weird files or rare formats, we could diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index b29c7becdf..2add2f733c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -60,7 +60,7 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; /** * 7Zip ingest module extracts supported archives, adds extracted DerivedFiles, @@ -92,7 +92,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F private static final int ZIP_SIGNATURE_BE = 0x504B0304; private IngestJobContext context; private long jobId; - private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private final static IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); SevenZipIngestModule() { } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index 1767714031..e623b4f561 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -43,7 +43,7 @@ import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.FileIngestModule; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.datamodel.HashInfo; public class HashDbIngestModule extends IngestModuleAdapter implements FileIngestModule { @@ -60,7 +60,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges private static AtomicLong totalKnownBadCount = new AtomicLong(0); private static AtomicLong totalCalctime = new AtomicLong(0); private static AtomicLong totalLookuptime = new AtomicLong(0); - private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); HashDbIngestModule(HashLookupModuleSettings settings) { this.settings = settings; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 5f4c0d34a7..dbb9c198e5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -38,7 +38,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestJobContext; -import org.sleuthkit.autopsy.ingest.ModuleReferenceCounter; +import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ReadContentInputStream; @@ -92,7 +92,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme private long dataSourceId; private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging private int instanceNum = 0; - private static ModuleReferenceCounter refCounter = new ModuleReferenceCounter(); + private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); private enum IngestStatus { From eaf2bd4f7ab0020f9926996bdffd69b4c921a300 Mon Sep 17 00:00:00 2001 From: "Samuel H. Kenyon" Date: Fri, 11 Apr 2014 14:57:24 -0400 Subject: [PATCH 6/6] added a shutdown() to do counter decrements --- .../autopsy/modules/sevenzip/SevenZipIngestModule.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java index 2add2f733c..da517d7b39 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java @@ -186,6 +186,12 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F return ProcessResult.OK; } + @Override + public void shutDown(boolean ingestJobCancelled) { + // We don't need the value, but for cleanliness and consistency, -- it + refCounter.decrementAndGet(jobId); + } + private void sendNewFilesEvent(AbstractFile archive, List unpackedFiles) { //currently sending a single event for all new files services.fireModuleContentEvent(new ModuleContentEvent(archive));
").append(FileTypeIdModuleFactory.getModuleName()).append("