From 4624a98c8c0b897fb72f6b7cf3cc873bd3b0ae2c Mon Sep 17 00:00:00 2001 From: adam-m Date: Wed, 25 Apr 2012 16:59:26 -0400 Subject: [PATCH 1/7] Keyword search ingest - simplify timer - use Timer utility instead implementing own --- .../KeywordSearchIngestService.java | 47 +++++++------------ 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java index 3d18c75c67..3cd25a6770 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java @@ -18,6 +18,8 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -27,6 +29,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; +import javax.swing.Timer; import org.apache.commons.lang.StringEscapeUtils; import org.apache.solr.client.solrj.SolrServerException; import org.netbeans.api.progress.ProgressHandle; @@ -45,7 +48,6 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.FsContent; import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskException; //service provider registered in layer.xml public final class KeywordSearchIngestService implements IngestServiceFsContent { @@ -59,12 +61,11 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent private static final long MAX_INDEX_SIZE = 100 * (1 << 10) * (1 << 10); private Ingester ingester = null; private volatile boolean commitIndex = false; //whether to commit index next time - private volatile boolean runTimer = false; private List keywords; //keywords to search private List keywordLists; // lists currently being searched private Map keywordToList; //keyword to list name mapping //private final Object lock = new Object(); - private Thread timer; + private Timer commitTimer; private Indexer indexer; private Searcher searcher; private volatile boolean searcherDone = true; @@ -149,7 +150,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent } //logger.log(Level.INFO, "complete()"); - runTimer = false; + commitTimer.stop(); //handle case if previous search running //cancel it, will re-run after final commit @@ -185,7 +186,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent logger.log(Level.INFO, "stop()"); //stop timer - runTimer = false; + commitTimer.stop(); //stop searcher if (searcher != null) { searcher.cancel(true); @@ -254,12 +255,11 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent final int commitIntervalMs = managerProxy.getUpdateFrequency() * 60 * 1000; logger.log(Level.INFO, "Using refresh interval (ms): " + commitIntervalMs); - timer = new CommitTimer(commitIntervalMs); - runTimer = true; + commitTimer = new Timer(commitIntervalMs, new CommitTimerAction() ); initialized = true; - timer.start(); + commitTimer.start(); managerProxy.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, this, "Started")); } @@ -410,31 +410,16 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent } } - //CommitTimer wakes up every interval ms - //and sets a flag for indexer to commit after indexing next file - private class CommitTimer extends Thread { - - private final Logger logger = Logger.getLogger(CommitTimer.class.getName()); - private int interval; - - CommitTimer(int interval) { - this.interval = interval; - } + + //CommitTimerAction to run by commitTimer + //sets a flag for indexer to commit after indexing next file + private class CommitTimerAction implements ActionListener { + private final Logger logger = Logger.getLogger(CommitTimerAction.class.getName()); @Override - public void run() { - while (runTimer) { - try { - Thread.sleep(interval); - commitIndex = true; - logger.log(Level.INFO, "CommitTimer awake"); - } catch (InterruptedException e) { - break; - } - - } - commitIndex = false; - return; + public void actionPerformed(ActionEvent e) { + commitIndex = true; + logger.log(Level.INFO, "CommitTimer awake"); } } From 7bead8d3cc876a8f607b9e6e44ffb5d5250c0873 Mon Sep 17 00:00:00 2001 From: adam-m Date: Wed, 25 Apr 2012 17:20:11 -0400 Subject: [PATCH 2/7] update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 804b0e96ed..6a8036f2d1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ /KeywordSearch/release/solr/webapps/solr.war /DataModel/release/modules/ext/sqlite-jdbc-3.7.2.jar -/DataModel/release/modules/lib/zlib.dll \ No newline at end of file +/DataModel/release/modules/lib/zlib.dll +/branding_spear \ No newline at end of file From c9bb5199621059c9746edd837f40927ba4184139 Mon Sep 17 00:00:00 2001 From: adam-m Date: Wed, 25 Apr 2012 17:28:45 -0400 Subject: [PATCH 3/7] Different logging handler behavior for dev builds (>= warn become dialogs) and release builds (>=severe become dialogs) --- .../autopsy/coreutils/AutopsyExceptionHandler.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java b/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java index eadff4a81b..fe59ddf690 100644 --- a/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java +++ b/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java @@ -40,11 +40,19 @@ public class AutopsyExceptionHandler extends Handler { static final int WARNING_VALUE = Level.WARNING.intValue(); static final int SEVERE_VALUE = Level.SEVERE.intValue(); static final Handler nbErrorManager = new NbErrorManager(); // Default NetBeans handler - + static final Version.Type buildType = Version.getBuildType(); + public AutopsyExceptionHandler() { super(); - // Only display messages for SEVERE level and above, that come from an uncaught exception. - this.setLevel(Level.SEVERE); + + if (buildType == Version.Type.DEVELOPMENT) + //for dev builds, show dialogs for WARNING and above + this.setLevel(Level.WARNING); + else + //for production builds, show dialogs for SEVERE and above (TODO in future consider not show any, explicit dialogs should be in place) + this.setLevel(Level.SEVERE); + + this.setFilter(new ExceptionFilter()); this.setFormatter(new SimpleFormatter()); } From 4dfdfa9af51169cae7706f31bfff956fcb0633ac Mon Sep 17 00:00:00 2001 From: adam-m Date: Wed, 25 Apr 2012 18:28:20 -0400 Subject: [PATCH 4/7] TSK-438 Handle out of disk space condition - add monitor that monitors system health (disk space) during ingest periodically, preliminary version --- .../autopsy/ingest/IngestManager.java | 6 + .../autopsy/ingest/IngestMonitor.java | 106 ++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 4f9c12fee0..d499b23a0c 100755 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -90,6 +90,9 @@ public class IngestManager { final IngestManagerProxy managerProxy = new IngestManagerProxy(this); //notifications private final static PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); + + //monitor + private final IngestMonitor ingestMonitor = new IngestMonitor(); private enum IngestManagerEvents { @@ -193,6 +196,9 @@ public class IngestManager { logger.log(Level.INFO, "Image queue: " + this.imageQueue.toString()); logger.log(Level.INFO, "File queue: " + this.fsContentQueue.toString()); + if (! ingestMonitor.isRunning()) + ingestMonitor.start(); + //image ingesters // cycle through each image in the queue while (hasNextImage()) { diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java new file mode 100644 index 0000000000..7453fb43e5 --- /dev/null +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -0,0 +1,106 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.ingest; + +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.io.File; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.swing.Timer; + + +/** + * Monitor health of the system and stop ingest if necessary + */ +public class IngestMonitor { + private static final int INITIAL_INTERVAL_MS = 60000; //1 min. + + private static final Logger logger = Logger.getLogger(IngestMonitor.class.getName()); + + private Timer timer; + + /** + * Start the monitor + */ + void start() { + timer = new Timer(INITIAL_INTERVAL_MS, new MonitorAction()); + timer.start(); + } + + /** + * Stop the monitor + */ + void stop() { + if (timer != null) + timer.stop(); + } + + /** + * Check if the monitor is running + * @return true if the monitor is running, false otherwise + */ + boolean isRunning() { + return timer != null && timer.isRunning(); + } + + private class MonitorAction implements ActionListener { + private final static long MIN_FREE_DISK_SPACE = 100L * 1024 * 1024; //100MB + + @Override + public void actionPerformed(ActionEvent e) { + final IngestManager manager = IngestManager.getDefault(); + + //runs checks only if ingest is running + if (manager.isIngestRunning() == false) + return; + + if (checkDiskSpace() == false) { + //stop ingest if running + logger.log(Level.SEVERE, "Stopping ingest due to low disk space"); + manager.stopAll(); + manager.postMessage(IngestMessage.createManagerMessage("Stopping ingest due to low disk space", "Stopping ingest due to low disk space. Please ensure the system disk has at least 1GB free space (more for large images) and restart ingest.")); + } + } + + /** + * check disk space + * @return true if OK, false otherwise + */ + private boolean checkDiskSpace() { + //assume root partition + //TODO use better check, i.e. root partition + user partition + try to write temp file + File root = new File(File.separator); + long freeSpace; + try { + freeSpace = root.getFreeSpace(); + } + catch (SecurityException e) { + logger.log(Level.INFO, "Unable to check for free disk space (probably permission issue)", e); + return true; //OK + } + //logger.log(Level.INFO, "Checking free disk apce: " + freeSpace + " need: " + Long.toString(MIN_FREE_DISK_SPACE)); + return freeSpace > MIN_FREE_DISK_SPACE; + } + + } + + +} From 3a9a50849f48f9ec7e6e0b532a8a04b2cdf0195d Mon Sep 17 00:00:00 2001 From: adam-m Date: Thu, 26 Apr 2012 09:56:15 -0400 Subject: [PATCH 5/7] Log handler - use severe level for dialogs for all builds for now --- .../sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java b/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java index fe59ddf690..a3d62de5f1 100644 --- a/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java +++ b/CoreUtils/src/org/sleuthkit/autopsy/coreutils/AutopsyExceptionHandler.java @@ -45,13 +45,15 @@ public class AutopsyExceptionHandler extends Handler { public AutopsyExceptionHandler() { super(); + this.setLevel(Level.SEVERE); + /* if (buildType == Version.Type.DEVELOPMENT) //for dev builds, show dialogs for WARNING and above this.setLevel(Level.WARNING); else //for production builds, show dialogs for SEVERE and above (TODO in future consider not show any, explicit dialogs should be in place) this.setLevel(Level.SEVERE); - + */ this.setFilter(new ExceptionFilter()); this.setFormatter(new SimpleFormatter()); From 43a9c27c63772af936c9a6794bac80a3fb573078 Mon Sep 17 00:00:00 2001 From: adam-m Date: Thu, 26 Apr 2012 12:19:12 -0400 Subject: [PATCH 6/7] - tweaks to ingest disk space monitor - keyword search index error message: add info if file is deleted --- .../autopsy/ingest/IngestMonitor.java | 21 +++++++++--- .../KeywordSearchIngestService.java | 32 ++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index 7453fb43e5..260f8fccf3 100644 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -25,6 +25,7 @@ import java.io.File; import java.util.logging.Level; import java.util.logging.Logger; import javax.swing.Timer; +import org.sleuthkit.autopsy.casemodule.Case; /** @@ -63,6 +64,19 @@ public class IngestMonitor { private class MonitorAction implements ActionListener { private final static long MIN_FREE_DISK_SPACE = 100L * 1024 * 1024; //100MB + private File root = new File(File.separator); //default, roto dir where autopsy runs + + MonitorAction() { + //find drive where case is located + String caseDir = Case.getCurrentCase().getCaseDirectory(); + File curDir = new File(caseDir); + File tempF = null; + while ( (tempF = curDir.getParentFile()) != null) + curDir = tempF; + root = curDir; + //logger.log(Level.INFO, "Using case root: " + curDir.getAbsolutePath()); + + } @Override public void actionPerformed(ActionEvent e) { @@ -76,7 +90,7 @@ public class IngestMonitor { //stop ingest if running logger.log(Level.SEVERE, "Stopping ingest due to low disk space"); manager.stopAll(); - manager.postMessage(IngestMessage.createManagerMessage("Stopping ingest due to low disk space", "Stopping ingest due to low disk space. Please ensure the system disk has at least 1GB free space (more for large images) and restart ingest.")); + manager.postMessage(IngestMessage.createManagerMessage("Stopping ingest due to low disk space", "Stopping ingest due to low disk space. Please ensure the drive where Case is located has at least 1GB free space (more for large images) and restart ingest.")); } } @@ -85,15 +99,12 @@ public class IngestMonitor { * @return true if OK, false otherwise */ private boolean checkDiskSpace() { - //assume root partition - //TODO use better check, i.e. root partition + user partition + try to write temp file - File root = new File(File.separator); long freeSpace; try { freeSpace = root.getFreeSpace(); } catch (SecurityException e) { - logger.log(Level.INFO, "Unable to check for free disk space (probably permission issue)", e); + logger.log(Level.WARNING, "Unable to check for free disk space (permission issue)", e); return true; //OK } //logger.log(Level.INFO, "Checking free disk apce: " + freeSpace + " need: " + Long.toString(MIN_FREE_DISK_SPACE)); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java index 3cd25a6770..50febd1667 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java @@ -48,6 +48,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.FsContent; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskData; //service provider registered in layer.xml public final class KeywordSearchIngestService implements IngestServiceFsContent { @@ -255,7 +256,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent final int commitIntervalMs = managerProxy.getUpdateFrequency() * 60 * 1000; logger.log(Level.INFO, "Using refresh interval (ms): " + commitIntervalMs); - commitTimer = new Timer(commitIntervalMs, new CommitTimerAction() ); + commitTimer = new Timer(commitIntervalMs, new CommitTimerAction()); initialized = true; @@ -410,10 +411,10 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent } } - //CommitTimerAction to run by commitTimer //sets a flag for indexer to commit after indexing next file private class CommitTimerAction implements ActionListener { + private final Logger logger = Logger.getLogger(CommitTimerAction.class.getName()); @Override @@ -429,6 +430,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent private class Indexer { private final Logger logger = Logger.getLogger(Indexer.class.getName()); + private static final String DELETED_MSG = "The file is an unallocated or orphan file (deleted) and entire content is no longer recoverable. "; private boolean extractAndIngest(FsContent f) { boolean success = false; @@ -465,7 +467,13 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent } } + String deletedMessage = ""; + if ((fsContent.getMeta_flags() & (TskData.TSK_FS_META_FLAG_ENUM.ORPHAN.getMetaFlag() | TskData.TSK_FS_META_FLAG_ENUM.UNALLOC.getMetaFlag())) != 0) { + deletedMessage = DELETED_MSG; + } + if (ingestible == true) { + try { //logger.log(Level.INFO, "indexing: " + fsContent.getName()); ingester.ingest(fsContent); @@ -474,40 +482,40 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED); //try to extract strings boolean processed = processNonIngestible(fsContent); - postIngestibleErrorMessage(processed, fileName); + postIngestibleErrorMessage(processed, fileName, deletedMessage); } catch (Exception e) { ingestStatus.put(fsContent.getId(), IngestStatus.SKIPPED); //try to extract strings boolean processed = processNonIngestible(fsContent); - postIngestibleErrorMessage(processed, fileName); + + postIngestibleErrorMessage(processed, fileName, deletedMessage); } } else { boolean processed = processNonIngestible(fsContent); - postNonIngestibleErrorMessage(processed, fsContent); + postNonIngestibleErrorMessage(processed, fsContent, deletedMessage); } } - private void postNonIngestibleErrorMessage(boolean stringsExtracted, FsContent fsContent) { + private void postNonIngestibleErrorMessage(boolean stringsExtracted, FsContent fsContent, String deletedMessage) { String fileName = fsContent.getName(); if (!stringsExtracted) { if (fsContent.getSize() < MAX_STRING_EXTRACT_SIZE) { - managerProxy.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchIngestService.instance, "Error indexing strings: " + fileName, "Error encountered extracting string content from this file (of unsupported format). The file will not be included in the search results.
File: " + fileName)); - } - else { + managerProxy.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchIngestService.instance, "Error indexing strings: " + fileName, "Error encountered extracting string content from this file (of unsupported format). " + deletedMessage + "The file will not be included in the search results.
File: " + fileName)); + } else { managerProxy.postMessage(IngestMessage.createMessage(++messageID, IngestMessage.MessageType.INFO, KeywordSearchIngestService.instance, "Skipped indexing strings: " + fileName, "Skipped extracting string content from this file (of unsupported format) due to the file size. The file will not be included in the search results.
File: " + fileName)); } } } - private void postIngestibleErrorMessage(boolean stringsExtracted, String fileName) { + private void postIngestibleErrorMessage(boolean stringsExtracted, String fileName, String deletedMessage) { if (stringsExtracted) { - managerProxy.postMessage(IngestMessage.createWarningMessage(++messageID, KeywordSearchIngestService.instance, "Indexed strings only: " + fileName, "Error encountered extracting file content. Used string extraction to index strings for partial analysis on this file.
File: " + fileName)); + managerProxy.postMessage(IngestMessage.createWarningMessage(++messageID, KeywordSearchIngestService.instance, "Indexed strings only: " + fileName, "Error encountered extracting file content. " + deletedMessage + "Used string extraction to index strings for partial analysis on this file.
File: " + fileName)); } else { - managerProxy.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchIngestService.instance, "Error indexing: " + fileName, "Error encountered extracting file content and strings from this file. The file will not be included in the search results.
File: " + fileName)); + managerProxy.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchIngestService.instance, "Error indexing: " + fileName, "Error encountered extracting file content and strings from this file. " + deletedMessage + "The file will not be included in the search results.
File: " + fileName)); } } From f760fa03c68dce38fbe3b82b4bbfab04fab3ab5f Mon Sep 17 00:00:00 2001 From: adam-m Date: Thu, 26 Apr 2012 12:42:09 -0400 Subject: [PATCH 7/7] ingest monitor: add disk drive path to the user message --- Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index 260f8fccf3..06ae82982f 100644 --- a/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Ingest/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -88,9 +88,10 @@ public class IngestMonitor { if (checkDiskSpace() == false) { //stop ingest if running - logger.log(Level.SEVERE, "Stopping ingest due to low disk space"); + final String diskPath = root.getAbsolutePath(); + logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath); manager.stopAll(); - manager.postMessage(IngestMessage.createManagerMessage("Stopping ingest due to low disk space", "Stopping ingest due to low disk space. Please ensure the drive where Case is located has at least 1GB free space (more for large images) and restart ingest.")); + manager.postMessage(IngestMessage.createManagerMessage("Stopping ingest due to low disk space on disk " + diskPath, "Stopping ingest due to low disk space on disk " + diskPath + ". Please ensure the drive where Case is located has at least 1GB free space (more for large images) and restart ingest.")); } }