diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardPanel3.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardPanel3.java index 2056d740c1..f1f3684d69 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardPanel3.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardPanel3.java @@ -34,6 +34,7 @@ import org.openide.WizardDescriptor; import org.openide.util.HelpCtx; import org.openide.util.Lookup; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitJNI.CaseDbHandle.AddImageProcess; @@ -381,6 +382,9 @@ class AddImageWizardPanel3 implements WizardDescriptor.Panel { // task cleanupImage.disable(); settings.putProperty(AddImageAction.IMAGECLEANUPTASK_PROP, null); + + logger.log(Level.INFO, "Image committed, imageId: " + imageId); + logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); } } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java index 834e03955c..43b7c150cd 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/PlatformUtil.java @@ -26,12 +26,15 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import javax.swing.filechooser.FileSystemView; import org.hyperic.sigar.Sigar; -import org.hyperic.sigar.SigarLoader; import org.openide.modules.InstalledFileLocator; import org.openide.modules.Places; import org.sleuthkit.autopsy.casemodule.LocalDisk; @@ -50,6 +53,7 @@ public class PlatformUtil { public static final String OS_ARCH_UNKNOWN = "unknown"; private static volatile long pid = -1; private static volatile Sigar sigar = null; + private static volatile MemoryMXBean memoryManager = null; /** * Get root path where the application is installed @@ -403,8 +407,7 @@ public class PlatformUtil { } if (sigar != null) { pid = sigar.getPid(); - } - else { + } else { System.out.println("Can't get PID"); } } catch (Exception e) { @@ -427,7 +430,7 @@ public class PlatformUtil { if (sigar == null) { sigar = org.sleuthkit.autopsy.corelibs.SigarLoader.getSigar(); } - + if (sigar == null || pid == -1) { System.out.println("Can't get virt mem used"); return -1; @@ -439,4 +442,51 @@ public class PlatformUtil { return virtMem; } + + /** + * Return formatted string with Jvm heap and non-heap memory usage + * + * @return formatted string with jvm memory usage + */ + public static String getJvmMemInfo() { + synchronized (PlatformUtil.class) { + if (memoryManager == null) { + memoryManager = ManagementFactory.getMemoryMXBean(); + } + } + final MemoryUsage heap = memoryManager.getHeapMemoryUsage(); + final MemoryUsage nonHeap = memoryManager.getNonHeapMemoryUsage(); + + return "JVM heap usage: " + heap.toString() + ", JVM non-heap usage: " + nonHeap.toString(); + + + } + + /** + * Return formatted string with physical memory usage + * + * @return formatted string with physical memory usage + */ + public static String getPhysicalMemInfo() { + final Runtime runTime = Runtime.getRuntime(); + final long maxMemory = runTime.maxMemory(); + final long totalMemory = runTime.totalMemory(); + final long freeMemory = runTime.freeMemory(); + return "Physical memory usage (max, total, free): " + + Long.toString(maxMemory) + ", " + Long.toString(totalMemory) + + ", " + Long.toString(freeMemory); + } + + /** + * Return formatted string with all memory usage (jvm, physical, native) + * + * @return formatted string with all memory usage info + */ + public static String getAllMemUsageInfo() { + StringBuilder sb = new StringBuilder(); + sb.append(PlatformUtil.getPhysicalMemInfo()).append("\n"); + sb.append(PlatformUtil.getJvmMemInfo()).append("\n"); + sb.append("Process Virtual Memory: ").append(PlatformUtil.getProcessVirtualMemoryUsed()); + return sb.toString(); + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index 2bec6c5308..365a07902c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -24,9 +24,6 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.io.File; import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryUsage; import java.util.logging.FileHandler; import java.util.logging.Level; import java.util.logging.SimpleFormatter; @@ -45,7 +42,6 @@ public class IngestMonitor { private final Logger logger = Logger.getLogger(IngestMonitor.class.getName()); private Timer timer; private static final java.util.logging.Logger MONITOR_LOGGER = java.util.logging.Logger.getLogger("monitor"); - private final MemoryMXBean memoryManager = ManagementFactory.getMemoryMXBean(); private MonitorAction monitor; IngestMonitor() { @@ -103,8 +99,7 @@ public class IngestMonitor { long getFreeSpace() { try { return monitor.getFreeSpace(); - } - catch (SecurityException e) { + } catch (SecurityException e) { logger.log(Level.WARNING, "Error checking for free disk space on ingest data drive", e); return -1; } @@ -204,21 +199,7 @@ public class IngestMonitor { * Monitor memory usage and print to memory log */ private void monitorMemory() { - - final Runtime runTime = Runtime.getRuntime(); - final long maxMemory = runTime.maxMemory(); - final long totalMemory = runTime.totalMemory(); - final long freeMemory = runTime.freeMemory(); - MONITOR_LOGGER.log(Level.INFO, "Physical memory (max, total, free): " - + Long.toString(maxMemory) + ", " + Long.toString(totalMemory) - + ", " + Long.toString(freeMemory)); - - final MemoryUsage heap = memoryManager.getHeapMemoryUsage(); - final MemoryUsage nonHeap = memoryManager.getNonHeapMemoryUsage(); - - MONITOR_LOGGER.log(Level.INFO, "Java heap memory: " + heap.toString() + ", Java non-heap memory: " + nonHeap.toString()); - MONITOR_LOGGER.log(Level.INFO, "Process Virtual Memory: " + PlatformUtil.getProcessVirtualMemoryUsed()); - + MONITOR_LOGGER.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); } } } diff --git a/CoreLibs/src/org/sleuthkit/autopsy/corelibs/SigarLoader.java b/CoreLibs/src/org/sleuthkit/autopsy/corelibs/SigarLoader.java index 876fa2c8fb..d2e0825d51 100644 --- a/CoreLibs/src/org/sleuthkit/autopsy/corelibs/SigarLoader.java +++ b/CoreLibs/src/org/sleuthkit/autopsy/corelibs/SigarLoader.java @@ -16,41 +16,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.corelibs; import java.io.File; import org.hyperic.sigar.Sigar; /** - * Wrapper over Sigar instrumentation class to facilitate dll loading. - * Our setup bypasses Sigar library loader which does not work well for netbeans environment - * We are responsible for loading the library ourselves. + * Wrapper over Sigar instrumentation class to facilitate dll loading. Our setup + * bypasses Sigar library loader which does not work well for netbeans + * environment We are responsible for loading the library ourselves. */ public class SigarLoader { private static volatile Sigar sigar; - + static { //bypass the process of validation/loading of the library by sigar jar - System.setProperty("org.hyperic.sigar.path", "-"); - //System.setProperty(org.hyperic.sigar.SigarLoader.PROP_SIGAR_JAR_NAME, "sigar-1.6.4.jar"); + System.setProperty("org.hyperic.sigar.path", "-"); + //System.setProperty(org.hyperic.sigar.SigarLoader.PROP_SIGAR_JAR_NAME, "sigar-1.6.4.jar"); } public static Sigar getSigar() { - if (sigar == null) { - synchronized (SigarLoader.class) { - + synchronized (SigarLoader.class) { + if (sigar == null) { try { //rely on netbeans / jna to locate the lib variation for architecture/OS - System.loadLibrary("libsigar"); + System.loadLibrary("libsigar"); sigar = new Sigar(); sigar.enableLogging(false); //forces a test } catch (UnsatisfiedLinkError ex) { System.out.println("Error loading sigar library" + ex.toString()); - } - catch (Exception ex) { + } catch (Exception ex) { System.out.println("Error loading sigar library" + ex.toString()); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 9db4f61ca1..88fedb821b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -229,7 +229,7 @@ public class Server { InputStream stream; OutputStream out; - boolean doRun = true; + volatile boolean doRun = true; InputStreamPrinterThread(InputStream stream, String type) { this.stream = stream; @@ -266,9 +266,11 @@ public class Server { public void run() { InputStreamReader isr = new InputStreamReader(stream); BufferedReader br = new BufferedReader(isr); + OutputStreamWriter osw = null; + BufferedWriter bw = null; try { - OutputStreamWriter osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset()); - BufferedWriter bw = new BufferedWriter(osw); + osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset()); + bw = new BufferedWriter(osw); String line = null; while (doRun && (line = br.readLine()) != null) { bw.write(line); @@ -280,7 +282,16 @@ public class Server { } bw.flush(); } catch (IOException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.WARNING, "Error redirecting Solr output stream"); + } + finally { + if (bw != null) { + try { + bw.close(); + } catch (IOException ex) { + logger.log(Level.WARNING, "Error closing Solr output stream"); + } + } } } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java index cffd3684cc..9ee57c7d61 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java @@ -51,6 +51,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; /** * Chrome recent activity extraction @@ -107,54 +108,69 @@ public class Chrome extends Extract implements IngestModuleImage { } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error when trying to get Chrome history files.", ex); } + + // get only the allocated ones, for now + List allocatedHistoryFiles = new ArrayList<>(); + for (FsContent historyFile : historyFiles) { + if (historyFile.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)) { + allocatedHistoryFiles.add(historyFile); + } + } + + // we should have only one allocated history file. Log a warning if we + // have more, but process them all + if (allocatedHistoryFiles.size() > 1) { + logger.log(Level.INFO, "Found more than one allocated Chrome history file. Processing them all."); + } else if (allocatedHistoryFiles.size() == 0) { + logger.log(Level.INFO, "Could not find an allocated Chrome history file."); + return; + } int j = 0; - if (historyFiles != null && !historyFiles.isEmpty()) { - while (j < historyFiles.size()) { - String temps = currentCase.getTempDirectory() + File.separator + historyFiles.get(j).getName().toString() + j + ".db"; - int errors = 0; - final FsContent historyFile = historyFiles.get(j++); - if (historyFile.getSize() == 0) { - continue; - } - try { - ContentUtils.writeToFile(historyFile, new File(temps)); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error writing temp sqlite db for Chrome web history artifacts.{0}", ex); - this.addErrorMessage(this.getName() + ": Error while trying to analyze file:" + historyFile.getName()); - } - File dbFile = new File(temps); - if (controller.isCancelled()) { - dbFile.delete(); - break; - } - List> tempList = null; - tempList = this.dbConnect(temps, chquery); - logger.log(Level.INFO, moduleName + "- Now getting history from " + temps + " with " + tempList.size() + "artifacts identified."); - for (HashMap result : tempList) { - - Collection bbattributes = new ArrayList(); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), "Recent Activity", ((result.get("url").toString() != null) ? result.get("url").toString() : ""))); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "Recent Activity", ((result.get("url").toString() != null) ? EscapeUtil.decodeURL(result.get("url").toString()) : ""))); - //TODO Revisit usage of deprecated constructor per TSK-583 - //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LAST_ACCESSED.getTypeID(), "Recent Activity", "Last Visited", ((Long.valueOf(result.get("last_visit_time").toString())) / 10000000))); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(), "Recent Activity", ((Long.valueOf(result.get("last_visit_time").toString())) / 10000000))); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER.getTypeID(), "Recent Activity", ((result.get("from_visit").toString() != null) ? result.get("from_visit").toString() : ""))); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), "Recent Activity", ((result.get("title").toString() != null) ? result.get("title").toString() : ""))); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), "Recent Activity", "Chrome")); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(), "Recent Activity", (Util.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : "")))); - this.addArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY, historyFile, bbattributes); - - } - if (errors > 0) { - this.addErrorMessage(this.getName() + ": Error parsing " + errors + " Chrome web history artifacts."); - } - + while (j < historyFiles.size()) { + String temps = currentCase.getTempDirectory() + File.separator + historyFiles.get(j).getName().toString() + j + ".db"; + int errors = 0; + final FsContent historyFile = historyFiles.get(j++); + if (historyFile.getSize() == 0) { + continue; + } + try { + ContentUtils.writeToFile(historyFile, new File(temps)); + } catch (IOException ex) { + logger.log(Level.SEVERE, "Error writing temp sqlite db for Chrome web history artifacts.{0}", ex); + this.addErrorMessage(this.getName() + ": Error while trying to analyze file:" + historyFile.getName()); + } + File dbFile = new File(temps); + if (controller.isCancelled()) { dbFile.delete(); + break; + } + List> tempList = null; + tempList = this.dbConnect(temps, chquery); + logger.log(Level.INFO, moduleName + "- Now getting history from " + temps + " with " + tempList.size() + "artifacts identified."); + for (HashMap result : tempList) { + + Collection bbattributes = new ArrayList(); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL.getTypeID(), "Recent Activity", ((result.get("url").toString() != null) ? result.get("url").toString() : ""))); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED.getTypeID(), "Recent Activity", ((result.get("url").toString() != null) ? EscapeUtil.decodeURL(result.get("url").toString()) : ""))); + //TODO Revisit usage of deprecated constructor per TSK-583 + //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LAST_ACCESSED.getTypeID(), "Recent Activity", "Last Visited", ((Long.valueOf(result.get("last_visit_time").toString())) / 10000000))); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(), "Recent Activity", ((Long.valueOf(result.get("last_visit_time").toString())) / 10000000))); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER.getTypeID(), "Recent Activity", ((result.get("from_visit").toString() != null) ? result.get("from_visit").toString() : ""))); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), "Recent Activity", ((result.get("title").toString() != null) ? result.get("title").toString() : ""))); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), "Recent Activity", "Chrome")); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN.getTypeID(), "Recent Activity", (Util.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : "")))); + this.addArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY, historyFile, bbattributes); + + } + if (errors > 0) { + this.addErrorMessage(this.getName() + ": Error parsing " + errors + " Chrome web history artifacts."); } - services.fireModuleDataEvent(new ModuleDataEvent("Recent Activity", BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY)); + dbFile.delete(); } + + services.fireModuleDataEvent(new ModuleDataEvent("Recent Activity", BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY)); } private void getBookmark(Image image, IngestImageWorkerController controller) {