From ae1793bfd3a19e4a35c2731a91930f3a73804a3d Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 29 Apr 2014 17:49:12 -0400 Subject: [PATCH 1/5] Safekeep ingest framework improvements --- .../autopsy/ingest/Bundle.properties | 6 - .../autopsy/ingest/Bundle_ja.properties | 130 ++- .../ingest/DataSourceIngestPipeline.java | 4 +- .../autopsy/ingest/DataSourceIngestTask.java | 44 ++ .../ingest/DataSourceIngestTaskScheduler.java | 57 ++ .../autopsy/ingest/FileIngestTask.java | 93 +++ .../ingest/FileIngestTaskScheduler.java | 418 ++++++++++ .../autopsy/ingest/GetFilesCountVisitor.java | 99 +++ .../sleuthkit/autopsy/ingest/IngestJob.java | 296 ++++--- .../autopsy/ingest/IngestJobContext.java | 4 +- .../autopsy/ingest/IngestManager.java | 496 ++++++------ .../ingest/IngestMessageTopComponent.java | 2 +- .../autopsy/ingest/IngestMonitor.java | 2 +- .../autopsy/ingest/IngestScheduler.java | 744 ------------------ .../autopsy/ingest/IngestServices.java | 4 +- 15 files changed, 1203 insertions(+), 1196 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java delete mode 100644 Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index fed46c8bac..144c7359fd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -52,12 +52,6 @@ Ensure the Case drive has at least 1GB free space and restart ingest. IngestJobConfigurationPanel.advancedButton.text=Advanced IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\: -IngestScheduler.FileSched.toString.curFiles.text=\ -CurFiles, size\: -IngestScheduler.FileSched.toString.curDirs.text=\ -CurDirs(stack), size\: -IngestScheduler.FileSched.toString.rootDirs.text=\ -RootDirs(sorted), size\: IngestManager.StartIngestJobsTask.run.displayName=Queueing ingest tasks IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...) IngestManager.StartIngestJobsTask.run.catchException.msg=An error occurred while starting ingest. Results may only be partial diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties index 69eee4e67e..e3433daff3 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties @@ -1,74 +1,68 @@ -CTL_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8 -HINT_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8\u30A6\u30A3\u30F3\u30C9\u30A6 -IngestDialog.closeButton.title=\u9589\u3058\u308B -IngestDialog.startButton.title=\u30B9\u30BF\u30FC\u30C8 -IngestDialog.title.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB -IngestJob.progress.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 -IngestJob.progress.dataSourceIngest.displayName={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 -IngestJob.progress.fileIngest.displayName={0}\u306E\u30D5\u30A1\u30A4\u30EB\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 -IngestJobConfigurationPanel.advancedButton.actionCommand=\u30A2\u30C9\u30D0\u30F3\u30B9 -IngestJobConfigurationPanel.advancedButton.text=\u30A2\u30C9\u30D0\u30F3\u30B9 -IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u306E\u51E6\u7406 -IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524A\u9664\u3055\u308C\u305F\u30D5\u30A1\u30A4\u30EB\u7B49\u306E\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u51E6\u7406\u3002\u3088\u308A\u5B8C\u5168\u306A\u7D50\u679C\u304C\u51FA\u307E\u3059\u304C\u3001\u5927\u304D\u3044\u30A4\u30E1\u30FC\u30B8\u3067\u306F\u51E6\u7406\u6642\u9593\u304C\u9577\u304F\u306A\u308B\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 -IngestManager.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC -IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 -IngestManager.StartIngestJobsTask.run.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 -IngestManager.StartIngestJobsTask.run.catchException.msg=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u306E\u958B\u59CB\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002\u7D50\u679C\u304C\u4E00\u90E8\u306E\u3082\u306E -IngestManager.StartIngestJobsTask.run.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF\u3092\u30AD\u30E5\u30FC\u30A4\u30F3\u30B0 -IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30BD\u30FC\u30B9\u3001\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u3001\u8A73\u7D30\u304A\u3088\u3073\u30C7\u30FC\u30BF\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 -IngestMessage.exception.srcSubjNotNull.msg=\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 -IngestMessage.exception.typeSrcSubjNotNull.msg=\u30E1\u30C3\u30BB\u30FC\u30B8\u30BF\u30A4\u30D7\u3001\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 -IngestMessage.toString.data.text=\ \u30C7\u30FC\u30BF\uFF1A{0} -IngestMessage.toString.date.text=\ \u65E5\u4ED8\uFF1A{0} -IngestMessage.toString.details.text=\ \u8A73\u7D30\uFF1A{0} -IngestMessage.toString.subject.text=\ \u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\uFF1A{0} -IngestMessage.toString.type.text=\u30BF\u30A4\u30D7\uFF1A{0} -IngestMessageDetailsPanel.copyMenuItem.text=\u30B3\u30D4\u30FC -IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30C6\u30AD\u30B9\u30C8\uFF0Fhtml -IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629E -IngestMessageDetailsPanel.viewArtifactButton.text=\u7D50\u679C\u3078\u79FB\u52D5 -IngestMessageDetailsPanel.viewContentButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u3078\u79FB\u52D5 -IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30D6\u30FC\u30EB\u5024\u3067\u306F\u306A\u3044\u3082\u306E\u306BBooleanRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F -IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65E5\u4ED8\u3067\u306F\u306A\u3044\u3082\u306E\u306BDateRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F\u3002 -IngestMessagePanel.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC -IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 -IngestMessagePanel.MsgTableMod.colNames.module=\u30E2\u30B8\u30E5\u30FC\u30EB -IngestMessagePanel.MsgTableMod.colNames.new=\u65B0\u898F\uFF1F -IngestMessagePanel.MsgTableMod.colNames.num=\u756A\u53F7 -IngestMessagePanel.MsgTableMod.colNames.subject=\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8 -IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30BF\u30A4\u30E0\u30B9\u30BF\u30F3\u30D7 -IngestMessagePanel.sortByComboBox.model.priority=\u512A\u5148\u5EA6 +CTL_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8 +HINT_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8\u30a6\u30a3\u30f3\u30c9\u30a6 +IngestDialog.closeButton.title=\u9589\u3058\u308b +IngestDialog.startButton.title=\u30b9\u30bf\u30fc\u30c8 +IngestDialog.title.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb +IngestJob.progress.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09 +IngestJob.progress.dataSourceIngest.displayName={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8 +IngestJob.progress.fileIngest.displayName={0}\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8 +IngestJobConfigurationPanel.advancedButton.actionCommand=\u30a2\u30c9\u30d0\u30f3\u30b9 +IngestJobConfigurationPanel.advancedButton.text=\u30a2\u30c9\u30d0\u30f3\u30b9 +IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u306e\u51e6\u7406 +IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524a\u9664\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb\u7b49\u306e\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u3092\u51e6\u7406\u3002\u3088\u308a\u5b8c\u5168\u306a\u7d50\u679c\u304c\u51fa\u307e\u3059\u304c\u3001\u5927\u304d\u3044\u30a4\u30e1\u30fc\u30b8\u3067\u306f\u51e6\u7406\u6642\u9593\u304c\u9577\u304f\u306a\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 +IngestManager.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc +IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 +IngestManager.StartIngestJobsTask.run.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09 +IngestManager.StartIngestJobsTask.run.catchException.msg=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u306e\u958b\u59cb\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002\u7d50\u679c\u304c\u4e00\u90e8\u306e\u3082\u306e +IngestManager.StartIngestJobsTask.run.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af\u3092\u30ad\u30e5\u30fc\u30a4\u30f3\u30b0 +IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30bd\u30fc\u30b9\u3001\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u3001\u8a73\u7d30\u304a\u3088\u3073\u30c7\u30fc\u30bf\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093 +IngestMessage.exception.srcSubjNotNull.msg=\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093 +IngestMessage.exception.typeSrcSubjNotNull.msg=\u30e1\u30c3\u30bb\u30fc\u30b8\u30bf\u30a4\u30d7\u3001\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093 +IngestMessage.toString.data.text=\ \u30c7\u30fc\u30bf\uff1a{0} +IngestMessage.toString.date.text=\ \u65e5\u4ed8\uff1a{0} +IngestMessage.toString.details.text=\ \u8a73\u7d30\uff1a{0} +IngestMessage.toString.subject.text=\ \u30b5\u30d6\u30b8\u30a7\u30af\u30c8\uff1a{0} +IngestMessage.toString.type.text=\u30bf\u30a4\u30d7\uff1a{0} +IngestMessageDetailsPanel.copyMenuItem.text=\u30b3\u30d4\u30fc +IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30c6\u30ad\u30b9\u30c8\uff0fhtml +IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629e +IngestMessageDetailsPanel.viewArtifactButton.text=\u7d50\u679c\u3078\u79fb\u52d5 +IngestMessageDetailsPanel.viewContentButton.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3078\u79fb\u52d5 +IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30d6\u30fc\u30eb\u5024\u3067\u306f\u306a\u3044\u3082\u306e\u306bBooleanRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f +IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65e5\u4ed8\u3067\u306f\u306a\u3044\u3082\u306e\u306bDateRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f\u3002 +IngestMessagePanel.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc +IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002 +IngestMessagePanel.MsgTableMod.colNames.module=\u30e2\u30b8\u30e5\u30fc\u30eb +IngestMessagePanel.MsgTableMod.colNames.new=\u65b0\u898f\uff1f +IngestMessagePanel.MsgTableMod.colNames.num=\u756a\u53f7 +IngestMessagePanel.MsgTableMod.colNames.subject=\u30b5\u30d6\u30b8\u30a7\u30af\u30c8 +IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30bf\u30a4\u30e0\u30b9\u30bf\u30f3\u30d7 +IngestMessagePanel.sortByComboBox.model.priority=\u512a\u5148\u5ea6 IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593 -IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uFF08\u6642\u7CFB\u5217\uFF09\u307E\u305F\u306F\u30E1\u30C3\u30BB\u30FC\u30B8\u306E\u512A\u5148\u5EA6\u3067\u30BD\u30FC\u30C8 -IngestMessagePanel.sortByLabel.text=\u4E0B\u8A18\u3067\u30BD\u30FC\u30C8\uFF1A -IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8A08\uFF1A +IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uff08\u6642\u7cfb\u5217\uff09\u307e\u305f\u306f\u30e1\u30c3\u30bb\u30fc\u30b8\u306e\u512a\u5148\u5ea6\u3067\u30bd\u30fc\u30c8 +IngestMessagePanel.sortByLabel.text=\u4e0b\u8a18\u3067\u30bd\u30fc\u30c8\uff1a +IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8a08\uff1a IngestMessagePanel.totalMessagesNameVal.text=- -IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30E6\u30CB\u30FC\u30AF\uFF1A +IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30e6\u30cb\u30fc\u30af\uff1a IngestMessagePanel.totalUniqueMessagesNameVal.text=- -IngestMessagesToolbar.customizeButton.toolTipText=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E1\u30C3\u30BB\u30FC\u30B8 -IngestMessageTopComponent.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 -IngestMessageTopComponent.displayReport.option.GenRpt=\u30EC\u30DD\u30FC\u30C8\u751F\u6210 +IngestMessagesToolbar.customizeButton.toolTipText=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e1\u30c3\u30bb\u30fc\u30b8 +IngestMessageTopComponent.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9 +IngestMessageTopComponent.displayReport.option.GenRpt=\u30ec\u30dd\u30fc\u30c8\u751f\u6210 IngestMessageTopComponent.displayReport.option.OK=OK -IngestMessageTopComponent.initComponents.name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 -IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30EC\u30DD\u30FC\u30C8 -IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30C7\u30A3\u30B9\u30AF{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3\u306E\u305F\u3081\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u4E2D\u6B62\u3057\u307E\u3059\u3002\ -\u30B1\u30FC\u30B9\u30C9\u30E9\u30A4\u30D6\u306B\u6700\u4F4E1GB\u306E\u7A7A\u304D\u9818\u57DF\u304C\u3042\u308B\u306E\u3092\u78BA\u8A8D\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u518D\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 -IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u304C\u4E2D\u6B62\u3055\u308C\u307E\u3057\u305F\u30FC{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3 -IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30B5\u30A4\u30BA\uFF1A -IngestScheduler.FileSched.toString.curDirs.text=\ -CurDirs(stack), \u30B5\u30A4\u30BA\uFF1A -IngestScheduler.FileSched.toString.curFiles.text=\ -CurFiles, \u30B5\u30A4\u30BA\uFF1A -IngestScheduler.FileSched.toString.rootDirs.text=\ -RootDirs(sorted), \u30B5\u30A4\u30BA\uFF1A -OpenIDE-Module-Name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 -IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF -IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF -IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF -IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF -IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30A8\u30E9\u30FC\uFF1A\ +IngestMessageTopComponent.initComponents.name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9 +IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30ec\u30dd\u30fc\u30c8 +IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3\u306e\u305f\u3081\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u4e2d\u6b62\u3057\u307e\u3059\u3002\ +\u30b1\u30fc\u30b9\u30c9\u30e9\u30a4\u30d6\u306b\u6700\u4f4e1GB\u306e\u7a7a\u304d\u9818\u57df\u304c\u3042\u308b\u306e\u3092\u78ba\u8a8d\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u518d\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002 +IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3 +IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30b5\u30a4\u30ba\uff1a +OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8 +IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af +IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af +IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af +IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af +IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30a8\u30e9\u30fc\uff1a\ \ {0} -IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uFF11\u3064\u307E\u305F\u306F\u8907\u6570\u306E\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u30B9\u30BF\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u306F\u30AD\u30E3\u30F3\u30BB\u30EB\u3055\u308C\u307E\u3057\u305F\u3002 -IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305F\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u7121\u52B9\u5316\u3059\u308B\u304B\u30A8\u30E9\u30FC\u3092\u89E3\u6C7A\u3057\u3001\u305D\u306E\u5F8C\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u53F3\u30AF\u30EA\u30C3\u30AF\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u5B9F\u884C\u3092\u9078\u629E\u3057\u3066\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u30EA\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 -IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u5931\u6557 \ No newline at end of file +IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uff11\u3064\u307e\u305f\u306f\u8907\u6570\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u30b9\u30bf\u30fc\u30c8\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\u3002\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u306f\u30ad\u30e3\u30f3\u30bb\u30eb\u3055\u308c\u307e\u3057\u305f\u3002 +IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305f\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u7121\u52b9\u5316\u3059\u308b\u304b\u30a8\u30e9\u30fc\u3092\u89e3\u6c7a\u3057\u3001\u305d\u306e\u5f8c\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u53f3\u30af\u30ea\u30c3\u30af\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u5b9f\u884c\u3092\u9078\u629e\u3057\u3066\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u30ea\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002 +IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u5931\u6557 \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index fb349f249c..b824ffe615 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -34,8 +34,8 @@ final class DataSourceIngestPipeline { private final List moduleTemplates; private List modules = new ArrayList<>(); - DataSourceIngestPipeline(IngestJob task, List moduleTemplates) { - this.job = task; + DataSourceIngestPipeline(IngestJob job, List moduleTemplates) { + this.job = job; this.moduleTemplates = moduleTemplates; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java new file mode 100755 index 0000000000..af27d9b8c0 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -0,0 +1,44 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.Content; + +final class DataSourceIngestTask { + + private final IngestJob ingestJob; + private final Content dataSource; + + DataSourceIngestTask(IngestJob ingestJob, Content dataSource) { + this.ingestJob = ingestJob; + this.dataSource = dataSource; + } + + IngestJob getIngestJob() { + return ingestJob; + } + + Content getDataSource() { + return dataSource; + } + + void execute() { + ingestJob.process(); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java new file mode 100755 index 0000000000..22a6124396 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java @@ -0,0 +1,57 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.concurrent.LinkedBlockingQueue; + +final class DataSourceIngestTaskScheduler { + + private static DataSourceIngestTaskScheduler instance = new DataSourceIngestTaskScheduler(); + private final LinkedBlockingQueue tasks = new LinkedBlockingQueue<>(); + + static DataSourceIngestTaskScheduler getInstance() { + return instance; + } + + private DataSourceIngestTaskScheduler() { + } + + synchronized void addTask(DataSourceIngestTask task) throws InterruptedException { + task.getIngestJob().notifyTaskPending(); + try { + tasks.put(task); + } + catch (InterruptedException ex) { + // RJCTOD: Need a safety notification to undo above + } + } + + DataSourceIngestTask getNextTask() throws InterruptedException { + return tasks.take(); + } + + boolean hasTasksForIngestJob(long jobId) { + for (DataSourceIngestTask task : tasks) { + if (task.getIngestJobId() == jobId) { + return true; + } + } + return false; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java new file mode 100755 index 0000000000..db0edf9edf --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -0,0 +1,93 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.Objects; +import java.util.logging.Level; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +final class FileIngestTask { + final AbstractFile file; + private final IngestJob ingestJob; + + FileIngestTask(AbstractFile file, IngestJob task) { + this.file = file; + this.ingestJob = task; + } + + public IngestJob getIngestJob() { + return ingestJob; + } + + public AbstractFile getFile() { + return file; + } + + void execute(long threadId) { + ingestJob.process(file); + } + + @Override + public String toString() { //RJCTODO: May not keep this + try { + return "ProcessTask{" + "file=" + file.getId() + ": " + file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}'; + } catch (TskCoreException ex) { + // RJCTODO +// FileIngestTaskScheduler.logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS + } + return "ProcessTask{" + "file=" + file.getId() + ": " + file.getName() + '}'; + } + + /** + * two process tasks are equal when the file/dir and modules are the + * same this enables are not to queue up the same file/dir, modules + * tuples into the root dir set + * + * @param obj + * @return + */ + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final FileIngestTask other = (FileIngestTask) obj; + if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { + return false; + } + IngestJob thisTask = this.getIngestJob(); + IngestJob otherTask = other.getIngestJob(); + if (thisTask != otherTask && (thisTask == null || !thisTask.equals(otherTask))) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int hash = 5; + hash = 47 * hash + Objects.hashCode(this.file); + hash = 47 * hash + Objects.hashCode(this.ingestJob); + return hash; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java new file mode 100755 index 0000000000..280012ebf8 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java @@ -0,0 +1,418 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.TreeSet; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DerivedFile; +import org.sleuthkit.datamodel.Directory; +import org.sleuthkit.datamodel.File; +import org.sleuthkit.datamodel.FileSystem; +import org.sleuthkit.datamodel.LayoutFile; +import org.sleuthkit.datamodel.LocalFile; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; +import org.sleuthkit.datamodel.VirtualDirectory; + +final class FileIngestTaskScheduler { + + private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); + private static FileIngestTaskScheduler instance; + private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); + private final List directoryTasks = new ArrayList<>(); + private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); // Unlimited capacity + private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); + + static synchronized FileIngestTaskScheduler getInstance() { + if (instance == null) { + instance = new FileIngestTaskScheduler(); + } + return instance; + } + + private FileIngestTaskScheduler() { + } + + synchronized void addTasks(IngestJob dataSourceTask, Content dataSource) { + Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); + List firstLevelFiles = new ArrayList<>(); + if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { + // The data source is file. + firstLevelFiles.add((AbstractFile) dataSource); + } else { + for (AbstractFile root : rootObjects) { + List children; + try { + children = root.getChildren(); + if (children.isEmpty()) { + //add the root itself, could be unalloc file, child of volume or image + firstLevelFiles.add(root); + } else { + //root for fs root dir, schedule children dirs/files + for (Content child : children) { + if (child instanceof AbstractFile) { + firstLevelFiles.add((AbstractFile) child); + } + } + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS + } + } + } + for (AbstractFile firstLevelFile : firstLevelFiles) { + FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask); + if (shouldEnqueueTask(fileTask)) { + rootDirectoryTasks.add(fileTask); + } + } + + // Reshuffle/update the dir and file level queues if needed + updateQueues(); + } + + synchronized void addTask(IngestJob ingestJob, AbstractFile file) { + try { + FileIngestTask fileTask = new FileIngestTask(file, ingestJob); + if (shouldEnqueueTask(fileTask)) { + fileTask.getIngestJob().notifyTaskPending(); + fileTasks.put(fileTask); // Queue has unlimited capacity, does not block. + } + } catch (InterruptedException ex) { + // RJCTODO: Perhaps this is the convenience method? + // RJCTODO: Need undo + } + } + + FileIngestTask getNextTask() throws InterruptedException { + FileIngestTask task = fileTasks.take(); + updateQueues(); + return task; + } + + synchronized boolean hasTasksForJob(long ingestJobId) { + for (FileIngestTask task : rootDirectoryTasks) { + if (task.getIngestJob().getJobId() == ingestJobId) { + return true; + } + } + + for (FileIngestTask task : directoryTasks) { + if (task.getIngestJob().getJobId() == ingestJobId) { + return true; + } + } + + for (FileIngestTask task : fileTasks) { + if (task.getIngestJob().getJobId() == ingestJobId) { + return true; + } + } + + return false; + } + + private void updateQueues() { + // we loop because we could have a directory that has all files + // that do not get enqueued + while (true) { + // There are files in the queue, we're done + if (fileTasks.isEmpty() == false) { + return; + } + // fill in the directory queue if it is empty. + if (this.directoryTasks.isEmpty()) { + // bail out if root is also empty -- we are done + if (rootDirectoryTasks.isEmpty()) { + return; + } + FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst(); + directoryTasks.add(rootTask); + } + //pop and push AbstractFile directory children if any + //add the popped and its leaf children onto cur file list + FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); + final AbstractFile parentFile = parentTask.file; + // add itself to the file list + if (shouldEnqueueTask(parentTask)) { + // RJCTODO + try { + parentTask.getIngestJob().notifyTaskPending(); + fileTasks.put(parentTask); + } catch (InterruptedException ex) { + // RJCTODO: Maybe make a convenience method + // RJCTODO: Need undo + } + } + // add its children to the file and directory lists + try { + List children = parentFile.getChildren(); + for (Content c : children) { + if (c instanceof AbstractFile) { + AbstractFile childFile = (AbstractFile) c; + FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getIngestJob()); + if (childFile.hasChildren()) { + this.directoryTasks.add(childTask); + } else if (shouldEnqueueTask(childTask)) { + // RJCTODO + try { + childTask.getIngestJob().notifyTaskPending(); + fileTasks.put(childTask); + } catch (InterruptedException ex) { + // RJCTODO: Maybe make a convenience method + // RJCTODO: Need undo + } + } + } + } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex); + } + } + } + + synchronized void emptyQueues() { // RJCTODO: Perhaps clear all... + this.rootDirectoryTasks.clear(); + this.directoryTasks.clear(); + this.fileTasks.clear(); + } + + /** + * Check if the file is a special file that we should skip + * + * @param processTask a task whose file to check if should be queued of + * skipped + * @return true if should be enqueued, false otherwise + */ + private static boolean shouldEnqueueTask(final FileIngestTask processTask) { + final AbstractFile aFile = processTask.file; + //if it's unalloc file, skip if so scheduled + if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { + return false; + } + String fileName = aFile.getName(); + if (fileName.equals(".") || fileName.equals("..")) { + return false; + } else if (aFile instanceof org.sleuthkit.datamodel.File) { + final org.sleuthkit.datamodel.File f = (File) aFile; + //skip files in root dir, starting with $, containing : (not default attributes) + //with meta address < 32, i.e. some special large NTFS and FAT files + FileSystem fs = null; + try { + fs = f.getFileSystem(); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS + } + TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP; + if (fs != null) { + fsType = fs.getFsType(); + } + if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { + //not fat or ntfs, accept all files + return true; + } + boolean isInRootDir = false; + try { + isInRootDir = f.getParentDirectory().isRoot(); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS + } + if (isInRootDir && f.getMetaAddr() < 32) { + String name = f.getName(); + if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) { + return false; + } + } else { + return true; + } + } + return true; + } + + /** + * Visitor that gets a collection of top level objects to be scheduled, such + * as root directories (if there is FS) or LayoutFiles and virtual + * directories, also if there is no FS. + */ + static class GetRootDirectoryVisitor extends GetFilesContentVisitor { + + @Override + public Collection visit(VirtualDirectory ld) { + //case when we hit a layout directoryor local file container, not under a real FS + //or when root virt dir is scheduled + Collection ret = new ArrayList<>(); + ret.add(ld); + return ret; + } + + @Override + public Collection visit(LayoutFile lf) { + //case when we hit a layout file, not under a real FS + Collection ret = new ArrayList<>(); + ret.add(lf); + return ret; + } + + @Override + public Collection visit(Directory drctr) { + //we hit a real directory, a child of real FS + Collection ret = new ArrayList<>(); + ret.add(drctr); + return ret; + } + + @Override + public Collection visit(FileSystem fs) { + return getAllFromChildren(fs); + } + + @Override + public Collection visit(File file) { + //can have derived files + return getAllFromChildren(file); + } + + @Override + public Collection visit(DerivedFile derivedFile) { + //can have derived files + //TODO test this and overall scheduler with derived files + return getAllFromChildren(derivedFile); + } + + @Override + public Collection visit(LocalFile localFile) { + //can have local files + //TODO test this and overall scheduler with local files + return getAllFromChildren(localFile); + } + } + + /** + * Root directory sorter + */ + private static class RootDirectoryTaskComparator implements Comparator { + + @Override + public int compare(FileIngestTask q1, FileIngestTask q2) { + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file); + AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file); + if (p1 == p2) { + return (int) (q2.file.getId() - q1.file.getId()); + } else { + return p2.ordinal() - p1.ordinal(); + } + } + + /** + * Priority determination for sorted AbstractFile, used by + * RootDirComparator + */ + private static class AbstractFilePriority { + + enum Priority { + + LAST, LOW, MEDIUM, HIGH + } + static final List LAST_PRI_PATHS = new ArrayList<>(); + static final List LOW_PRI_PATHS = new ArrayList<>(); + static final List MEDIUM_PRI_PATHS = new ArrayList<>(); + static final List HIGH_PRI_PATHS = new ArrayList<>(); + /* prioritize root directory folders based on the assumption that we are + * looking for user content. Other types of investigations may want different + * priorities. */ + + static /* prioritize root directory folders based on the assumption that we are + * looking for user content. Other types of investigations may want different + * priorities. */ { + // these files have no structure, so they go last + //unalloc files are handled as virtual files in getPriority() + //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); + //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE)); + LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE)); + LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE)); + // orphan files are often corrupt and windows does not typically have + // user content, so put them towards the bottom + LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE)); + LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE)); + // all other files go into the medium category too + MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE)); + // user content is top priority + HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE)); + HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE)); + HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE)); + HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE)); + } + + /** + * Get the scheduling priority for a given file. + * + * @param abstractFile + * @return + */ + static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) { + if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { + //quickly filter out unstructured content + //non-fs virtual files and dirs, such as representing unalloc space + return AbstractFilePriority.Priority.LAST; + } + //determine the fs files priority by name + final String path = abstractFile.getName(); + if (path == null) { + return AbstractFilePriority.Priority.MEDIUM; + } + for (Pattern p : HIGH_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.HIGH; + } + } + for (Pattern p : MEDIUM_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.MEDIUM; + } + } + for (Pattern p : LOW_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.LOW; + } + } + for (Pattern p : LAST_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.LAST; + } + } + //default is medium + return AbstractFilePriority.Priority.MEDIUM; + } + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java new file mode 100755 index 0000000000..346da5d418 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java @@ -0,0 +1,99 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.ContentVisitor; +import org.sleuthkit.datamodel.FileSystem; +import org.sleuthkit.datamodel.LayoutFile; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; + +/** + * Get counts of ingestable files/dirs for the content input source. + * + * Note, also includes counts of all unalloc children files (for the fs, image, + * volume) even if ingest didn't ask for them + */ +final class GetFilesCountVisitor extends ContentVisitor.Default { + + private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); + + @Override + public Long visit(FileSystem fs) { + //recursion stop here + //case of a real fs, query all files for it + SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase(); + StringBuilder queryB = new StringBuilder(); + queryB.append("( (fs_obj_id = ").append(fs.getId()); //NON-NLS + //queryB.append(") OR (fs_obj_id = NULL) )"); + queryB.append(") )"); + queryB.append(" AND ( (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS + queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS + queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS + queryB.append(") )"); + //queryB.append( "AND (type = "); + //queryB.append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.getFileType()); + //queryB.append(")"); + try { + final String query = queryB.toString(); + logger.log(Level.INFO, "Executing count files query: {0}", query); //NON-NLS + return sc.countFilesWhere(query); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); //NON-NLS + return 0L; + } + } + + @Override + public Long visit(LayoutFile lf) { + //recursion stop here + //case of LayoutFile child of Image or Volume + return 1L; + } + + private long getCountFromChildren(Content content) { + long count = 0; + try { + List children = content.getChildren(); + if (children.size() > 0) { + for (Content child : children) { + count += child.accept(this); + } + } else { + count = 1; + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Could not get count of objects from children to get num of total files to be ingested", ex); //NON-NLS + } + return count; + } + + @Override + protected Long defaultVisit(Content cntnt) { + //recurse assuming this is image/vs/volume + //recursion stops at fs or unalloc file + return getCountFromChildren(cntnt); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 6d41cac96a..8982be2061 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -19,12 +19,16 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandleFactory; import org.openide.util.Cancellable; import org.openide.util.NbBundle; +import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; /** @@ -33,31 +37,63 @@ import org.sleuthkit.datamodel.Content; */ final class IngestJob { - private final long id; + private static final AtomicLong nextIngestJobId = new AtomicLong(0L); + private static final ConcurrentHashMap ingestJobs = new ConcurrentHashMap<>(); // Maps job ids to jobs. + private final long jobId; private final Content dataSource; private final List ingestModuleTemplates; private final boolean processUnallocatedSpace; - private final HashMap fileIngestPipelines = new HashMap<>(); - private final HashMap dataSourceIngestPipelines = new HashMap<>(); - private final IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); - private FileIngestPipeline initialFileIngestPipeline = null; - private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; - private ProgressHandle dataSourceTaskProgress; + private final LinkedBlockingQueue dataSourceIngestPipelines = new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue fileIngestPipelines = new LinkedBlockingQueue<>(); + private final AtomicInteger tasksInProgress = new AtomicInteger(0); + private final AtomicLong processedFiles = new AtomicLong(0L); + private ProgressHandle dataSourceTasksProgress; private ProgressHandle fileTasksProgress; - int totalEnqueuedFiles = 0; - private int processedFiles = 0; + private long filesToIngestEstimate = 0; private volatile boolean cancelled; - IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { - this.id = id; + static List startIngestJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { // RJCTODO: return errors + long jobId = nextIngestJobId.incrementAndGet(); + IngestJob ingestJob = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); + List errors = ingestJob.start(); + if (errors.isEmpty()) { + ingestJobs.put(jobId, ingestJob); + } + return errors; + } + + static boolean jobsAreRunning() { + for (IngestJob job : ingestJobs.values()) { + if (!job.isCancelled()) { + return true; + } + } + return false; + } + + static void addFileToIngestJob(long ingestJobId, AbstractFile file) { // RJCTODO: Move back to IngestManager + IngestJob job = ingestJobs.get(ingestJobId); + if (job != null) { + FileIngestTaskScheduler.getInstance().addTask(job, file); + } + } + + static void cancelAllIngestJobs() { + for (IngestJob job : ingestJobs.values()) { + job.cancel(); + } + } + + private IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { + this.jobId = id; this.dataSource = dataSource; this.ingestModuleTemplates = ingestModuleTemplates; this.processUnallocatedSpace = processUnallocatedSpace; this.cancelled = false; } - long getId() { - return id; + long getJobId() { + return jobId; } Content getDataSource() { @@ -68,34 +104,78 @@ final class IngestJob { return processUnallocatedSpace; } - synchronized List startUpIngestPipelines() { - startDataSourceIngestProgressBar(); - startFileIngestProgressBar(); - return startUpInitialIngestPipelines(); + List start() { + List errors = startUpIngestPipelines(); + if (errors.isEmpty()) { + DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, dataSource)); + FileIngestTaskScheduler.getInstance().addTasks(this, dataSource); + startDataSourceIngestProgressBar(); + startFileIngestProgressBar(); + } + return errors; + } + + private List startUpIngestPipelines() { + List errors = new ArrayList<>(); + + int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads(); + for (int i = 0; i < maxNumberOfPipelines; ++i) { + DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); + errors.addAll(pipeline.startUp()); + try { + dataSourceIngestPipelines.put(pipeline); + } catch (InterruptedException ex) { + // RJCTODO: log unexpected block and interrupt, or throw + } + if (errors.isEmpty()) { + // No need to accumulate presumably redundant erros. + break; + } + } + + maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads(); + for (int i = 0; i < maxNumberOfPipelines; ++i) { + FileIngestPipeline pipeline = new FileIngestPipeline(this, ingestModuleTemplates); + errors.addAll(pipeline.startUp()); + try { + fileIngestPipelines.put(pipeline); + } catch (InterruptedException ex) { + // RJCTODO: log unexpected block and interrupt, or throw + } + if (errors.isEmpty()) { + // No need to accumulate presumably redundant erros. + break; + } + } + + return errors; } private void startDataSourceIngestProgressBar() { - final String displayName = NbBundle - .getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", this.dataSource.getName()); - dataSourceTaskProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.dataSourceIngest.displayName", + dataSource.getName()); + dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { - if (dataSourceTaskProgress != null) { - dataSourceTaskProgress.setDisplayName(NbBundle.getMessage(this.getClass(), + if (dataSourceTasksProgress != null) { + dataSourceTasksProgress.setDisplayName( + NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", displayName)); } - IngestManager.getInstance().cancelIngestJobs(); + IngestJob.this.cancel(); return true; } }); - dataSourceTaskProgress.start(); - dataSourceTaskProgress.switchToIndeterminate(); + dataSourceTasksProgress.start(); + dataSourceTasksProgress.switchToIndeterminate(); // RJCTODO: check out the logic in the pipleine class } private void startFileIngestProgressBar() { - final String displayName = NbBundle - .getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); + final String displayName = NbBundle.getMessage(this.getClass(), + "IngestJob.progress.fileIngest.displayName", + dataSource.getName()); fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { @@ -104,124 +184,90 @@ final class IngestJob { NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", displayName)); } - IngestManager.getInstance().cancelIngestJobs(); + IngestJob.this.cancel(); return true; } }); + filesToIngestEstimate = dataSource.accept(new GetFilesCountVisitor()); fileTasksProgress.start(); - fileTasksProgress.switchToIndeterminate(); - totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); - fileTasksProgress.switchToDeterminate(totalEnqueuedFiles); + fileTasksProgress.switchToDeterminate((int) filesToIngestEstimate); } - private List startUpInitialIngestPipelines() { - // Create a per thread instance of each pipeline type right now to make - // (reasonably) sure that the ingest modules can be started. - initialDataSourceIngestPipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); - initialFileIngestPipeline = new FileIngestPipeline(this, ingestModuleTemplates); - List errors = new ArrayList<>(); - errors.addAll(initialDataSourceIngestPipeline.startUp()); - errors.addAll(initialFileIngestPipeline.startUp()); - return errors; + /** + * Called by the ingest task schedulers when an ingest task for this ingest + * job is added to the scheduler's task queue. + */ + void notifyTaskScheduled() { + // Increment the task counter when a task is scheduled so that there is + // a persistent record of the task's existence even after it is removed + // from the scheduler by an ingest thread. The task counter is used by + // the job to determine when it is done. + tasksInProgress.incrementAndGet(); } - synchronized DataSourceIngestPipeline getDataSourceIngestPipelineForThread(long threadId) { - DataSourceIngestPipeline pipeline; - if (initialDataSourceIngestPipeline != null) { - pipeline = initialDataSourceIngestPipeline; - initialDataSourceIngestPipeline = null; - dataSourceIngestPipelines.put(threadId, pipeline); - } else if (!dataSourceIngestPipelines.containsKey(threadId)) { - pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); - pipeline.startUp(); - dataSourceIngestPipelines.put(threadId, pipeline); - } else { - pipeline = dataSourceIngestPipelines.get(threadId); - } - return pipeline; + /** + * Called by the ingest schedulers as an "undo" operation for + * notifyTaskScheduled(). + */ + void notifyTaskCompleted() { + // Decrement the task counter when a task is discarded by a scheduler. + // The task counter is used by the job to determine when it is done. + tasksInProgress.decrementAndGet(); } - synchronized FileIngestPipeline getFileIngestPipelineForThread(long threadId) { - FileIngestPipeline pipeline; - if (initialFileIngestPipeline != null) { - pipeline = initialFileIngestPipeline; - initialFileIngestPipeline = null; - fileIngestPipelines.put(threadId, pipeline); - } else if (!fileIngestPipelines.containsKey(threadId)) { - pipeline = new FileIngestPipeline(this, ingestModuleTemplates); - pipeline.startUp(); - fileIngestPipelines.put(threadId, pipeline); - } else { - pipeline = fileIngestPipelines.get(threadId); + void process() throws InterruptedException { + if (!isCancelled()) { + try { + DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take(); + pipeline.process(); // RJCTODO: Pass data source through? + dataSourceIngestPipelines.put(pipeline); + } catch (InterruptedException ex) { + // RJCTODO: + } } - return pipeline; + ifCompletedShutDown(); } - synchronized List releaseIngestPipelinesForThread(long threadId) { - List errors = new ArrayList<>(); - - DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId); - if (dataSourceIngestPipeline != null) { - errors.addAll(dataSourceIngestPipeline.shutDown(cancelled)); - dataSourceIngestPipelines.remove(threadId); + void process(AbstractFile file) { + if (!isCancelled()) { + try { + FileIngestPipeline pipeline = fileIngestPipelines.take(); + fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); + pipeline.process(file); + fileIngestPipelines.put(pipeline); + } catch (InterruptedException ex) { + // RJCTODO: Log block and interrupt + } } - if (initialDataSourceIngestPipeline == null && dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) { - dataSourceTaskProgress.finish(); - dataSourceTaskProgress = null; - } - - FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId); - if (fileIngestPipeline != null) { - errors.addAll(fileIngestPipeline.shutDown(cancelled)); - fileIngestPipelines.remove(threadId); - } - if (initialFileIngestPipeline == null && fileIngestPipelines.isEmpty() && fileTasksProgress != null) { - fileTasksProgress.finish(); - fileTasksProgress = null; - } - - return errors; + ifCompletedShutDown(); } - synchronized boolean areIngestPipelinesShutDown() { - return (initialDataSourceIngestPipeline == null - && dataSourceIngestPipelines.isEmpty() - && initialFileIngestPipeline == null - && fileIngestPipelines.isEmpty()); + void ifCompletedShutDown() { + if (tasksInProgress.decrementAndGet() == 0) { + while (!dataSourceIngestPipelines.isEmpty()) { + DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll(); + pipeline.shutDown(cancelled); + } + while (!fileIngestPipelines.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelines.poll(); + pipeline.shutDown(cancelled); + } + ingestJobs.remove(jobId); + IngestManager.getInstance().fireIngestJobCompleted(jobId); + } } - synchronized ProgressHandle getDataSourceTaskProgressBar() { - return this.dataSourceTaskProgress; - } - - synchronized void updateFileTasksProgressBar(String currentFileName) { - int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); - if (newTotalEnqueuedFiles > totalEnqueuedFiles) { - totalEnqueuedFiles = newTotalEnqueuedFiles + 1; - fileTasksProgress.switchToIndeterminate(); - fileTasksProgress.switchToDeterminate(totalEnqueuedFiles); - } - if (processedFiles < totalEnqueuedFiles) { - ++processedFiles; - } - - fileTasksProgress.progress(currentFileName, processedFiles); - } - - synchronized void cancel() { - if (initialDataSourceIngestPipeline != null) { - initialDataSourceIngestPipeline.shutDown(true); - initialDataSourceIngestPipeline = null; - } - if (initialFileIngestPipeline != null) { - initialFileIngestPipeline.shutDown(true); - initialFileIngestPipeline = null; - } - - cancelled = true; + ProgressHandle getDataSourceTaskProgressBar() { + return dataSourceTasksProgress; // RJCTODO: Should just pass the progress handle or the object to the pipeline } boolean isCancelled() { return cancelled; } + + void cancel() { + cancelled = true; + fileTasksProgress.finish(); + IngestManager.getInstance().fireIngestJobCancelled(jobId); + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index dc9fb58e3f..6a9b4a9454 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -39,7 +39,7 @@ public final class IngestJobContext { * @return The ingest job identifier. */ public long getJobId() { - return this.ingestJob.getId(); + return this.ingestJob.getJobId(); } /** @@ -60,7 +60,7 @@ public final class IngestJobContext { */ public void addFiles(List files) { for (AbstractFile file : files) { - IngestManager.getInstance().addFileToIngestJob(ingestJob.getId(), file); + IngestJob.addFileToIngestJob(ingestJob.getJobId(), file); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index d94ee00059..a5b0da296c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -20,9 +20,7 @@ package org.sleuthkit.autopsy.ingest; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; -import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -36,72 +34,146 @@ import org.netbeans.api.progress.ProgressHandleFactory; import org.openide.util.Cancellable; import org.openide.util.NbPreferences; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import java.util.prefs.Preferences; import javax.swing.JOptionPane; -import javax.swing.SwingWorker; -import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask; /** * Manages the execution of ingest jobs. */ public class IngestManager { + private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1; private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); - private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final IngestManager instance = new IngestManager(); - private final IngestScheduler scheduler = IngestScheduler.getInstance(); + private final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); private final IngestMonitor ingestMonitor = new IngestMonitor(); - private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(); - private final ExecutorService dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); - private final ExecutorService fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); - private final ExecutorService fireEventTasksExecutor = Executors.newSingleThreadExecutor(); - private final ConcurrentHashMap ingestJobs = new ConcurrentHashMap<>(1, 0.9f, 4); // Maps job ids to jobs. - private final ConcurrentHashMap> ingestTasks = new ConcurrentHashMap<>(); // Maps task ids to task cancellation handles. Guarded by this. - private final AtomicLong ingestJobId = new AtomicLong(0L); - private final AtomicLong ingestTaskId = new AtomicLong(0L); + private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); + private final ConcurrentHashMap> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. + private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); + private final ConcurrentHashMap> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. + private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); + private final ExecutorService fireIngestJobEventsThreadPool = Executors.newSingleThreadExecutor(); + private final ConcurrentHashMap> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. + private final AtomicLong nextThreadId = new AtomicLong(0L); private volatile IngestMessageTopComponent ingestMessageBox; /** - * Gets the IngestManager singleton, creating it if necessary. + * Gets the ingest manager. * - * @returns The IngestManager singleton. + * @returns A singleton IngestManager object. */ public static IngestManager getInstance() { return instance; } - private IngestManager() { - } - /** - * Signals to the ingest manager that it can go find the top component for - * the ingest messages in box. Called by the custom installer for this - * package once the window system is initialized. + * Starts the ingest monitor and the data source ingest and file ingest + * threads. */ - void initIngestMessageInbox() { - if (this.ingestMessageBox == null) { - this.ingestMessageBox = IngestMessageTopComponent.findInstance(); + private IngestManager() { + startDataSourceIngestThread(); + int numberOfFileIngestThreads = getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + startFileIngestThread(); } } + /** + * Signals to the ingest manager that it can go about finding the top + * component for the ingest messages in box. Called by the custom installer + * for this package once the window system is initialized. + */ + void initIngestMessageInbox() { + if (ingestMessageBox == null) { + ingestMessageBox = IngestMessageTopComponent.findInstance(); + } + } + + /** + * Gets the maximum number of data source ingest threads the ingest manager + * will use. + */ + public static int getMaxNumberOfDataSourceIngestThreads() { + return MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS; + } + + /** + * Gets the maximum number of file ingest threads the ingest manager will + * use. + */ + public static int getMaxNumberOfFileIngestThreads() { + return MAX_NUMBER_OF_FILE_INGEST_THREADS; + } + + /** + * Gets the number of file ingest threads the ingest manager will use. + */ public synchronized static int getNumberOfFileIngestThreads() { return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); } + /** + * Changes the number of file ingest threads the ingest manager will use to + * no more than MAX_NUMBER_OF_FILE_INGEST_THREADS and no less than + * MIN_NUMBER_OF_FILE_INGEST_THREADS. Out of range requests are converted to + * requests for DEFAULT_NUMBER_OF_FILE_INGEST_THREADS. + * + * @param numberOfThreads The desired number of file ingest threads. + */ public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) { - if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS - || numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) { + if ((numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) { numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; } - userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); + + if (instance.fileIngestThreads.size() != numberOfThreads) { + if (instance.fileIngestThreads.size() > numberOfThreads) { + Long[] threadIds = instance.fileIngestThreads.keySet().toArray(new Long[instance.fileIngestThreads.size()]); + int numberOfThreadsToCancel = instance.fileIngestThreads.size() - numberOfThreads; + for (int i = 0; i < numberOfThreadsToCancel; ++i) { + instance.cancelFileIngestThread(threadIds[i]); + } + } else if (instance.fileIngestThreads.size() < numberOfThreads) { + int numberOfThreadsToAdd = numberOfThreads - instance.fileIngestThreads.size(); + for (int i = 0; i < numberOfThreadsToAdd; ++i) { + instance.startFileIngestThread(); + } + } + } + } + + /** + * Submits a DataSourceIngestThread Runnable to the data source ingest + * thread pool. + */ + private void startDataSourceIngestThread() { + long threadId = nextThreadId.incrementAndGet(); + Future handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread(threadId)); + dataSourceIngestThreads.put(threadId, handle); + } + + /** + * Submits a DataSourceIngestThread Runnable to the data source ingest + * thread pool. + */ + private void startFileIngestThread() { + long threadId = nextThreadId.incrementAndGet(); + Future handle = fileIngestThreadPool.submit(new FileIngestThread(threadId)); + fileIngestThreads.put(threadId, handle); + } + + /** + * Cancels a DataSourceIngestThread Runnable in the file ingest thread pool. + */ + private void cancelFileIngestThread(long threadId) { + Future handle = fileIngestThreads.remove(threadId); + handle.cancel(true); } synchronized void startIngestJobs(final List dataSources, final List moduleTemplates, boolean processUnallocatedSpace) { @@ -109,9 +181,9 @@ public class IngestManager { ingestMessageBox.clearMessages(); } - long taskId = ingestTaskId.incrementAndGet(); - Future task = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); - ingestTasks.put(taskId, task); + long taskId = nextThreadId.incrementAndGet(); + Future task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); + fileIngestThreads.put(taskId, task); if (ingestMessageBox != null) { ingestMessageBox.restoreMessages(); @@ -121,43 +193,42 @@ public class IngestManager { /** * Test if any ingest jobs are in progress. * - * @return True if any ingest jobs are in progress, false otherwise + * @return True if any ingest jobs are in progress, false otherwise. */ public boolean isIngestRunning() { - return (ingestJobs.isEmpty() == false); + return IngestJob.jobsAreRunning(); } - void addFileToIngestJob(long ingestJobId, AbstractFile file) { - IngestJob job = ingestJobs.get(ingestJobId); - if (job != null) { - scheduler.getFileIngestScheduler().queueFile(job, file); + public void cancelAllIngestJobs() { + cancelStartIngestJobsTasks(); + IngestJob.cancelAllIngestJobs(); + } + + private void cancelStartIngestJobsTasks() { + for (Future future : startIngestJobThreads.values()) { + future.cancel(true); } - } - - void cancelIngestJobs() { - new IngestCancellationWorker().execute(); + startIngestJobThreads.clear(); } /** * Ingest events. */ - public enum IngestEvent { + public enum IngestEvent { // RJCTODO: Update comments if time permits /** - * Property change event fired when an ingest job is started. The ingest - * job id is in old value field of the PropertyChangeEvent object. + * Property change event fired when an ingest job is started. The old + * and new values of the PropertyChangeEvent object are set to null. */ INGEST_JOB_STARTED, /** - * Property change event fired when an ingest job is completed. The - * ingest job id is in old value field of the PropertyChangeEvent - * object. + * Property change event fired when an ingest job is completed. The old + * and new values of the PropertyChangeEvent object are set to null. */ INGEST_JOB_COMPLETED, /** - * Property change event fired when an ingest job is canceled. The - * ingest job id is in old value field of the PropertyChangeEvent - * object. + * Property change event fired when an ingest job is canceled. The old + * and new values of the PropertyChangeEvent object are set to null. */ INGEST_JOB_CANCELLED, /** @@ -182,79 +253,84 @@ public class IngestManager { }; /** - * Add property change listener to listen to ingest events. + * Add an ingest event property change listener. * - * @param listener PropertyChangeListener to register + * @param listener The PropertyChangeListener to register. */ - public static void addPropertyChangeListener(final PropertyChangeListener listener) { + public void addPropertyChangeListener(final PropertyChangeListener listener) { pcs.addPropertyChangeListener(listener); } - public static void removePropertyChangeListener(final PropertyChangeListener listener) { + /** + * Remove an ingest event property change listener. + * + * @param listener The PropertyChangeListener to unregister. + */ + public void removePropertyChangeListener(final PropertyChangeListener listener) { pcs.removePropertyChangeListener(listener); } - static void fireIngestJobEvent(String eventType, long jobId) { - try { - pcs.firePropertyChange(eventType, jobId, null); - } catch (Exception e) { - logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); - } + /** + * Fire an ingest event signifying an ingest job started. + * + * @param ingestJobId The ingest job id. + */ + void fireIngestJobStarted(long ingestJobId) { + fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED, ingestJobId, null)); } /** - * Fire event when file is done with a pipeline run + * Fire an ingest event signifying an ingest job finished. * - * @param fileId ID of file that is done + * @param ingestJobId The ingest job id. */ - static void fireFileIngestDone(long fileId) { - try { - pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null); - } catch (Exception e) { - logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); - } + void fireIngestJobCompleted(long ingestJobId) { + fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null)); } /** - * Fire event for ModuleDataEvent (when modules post data to blackboard, - * etc.) + * Fire an ingest event signifying an ingest job was canceled. * - * @param moduleDataEvent + * @param ingestJobId The ingest job id. */ - static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { - try { - pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null); - } catch (Exception e) { - logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); - } + void fireIngestJobCancelled(long ingestJobId) { + fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null)); } /** - * Fire event for ModuleContentChanged (when modules create new content that - * needs to be analyzed) + * Fire an ingest event signifying the ingest of a file is completed. * - * @param moduleContentEvent + * @param fileId The object id of file. */ - static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { - try { - pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null); - } catch (Exception e) { - logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), - MessageNotifyUtil.MessageType.ERROR); - } + void fireFileIngestDone(long fileId) { + fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.FILE_DONE, fileId, null)); } + /** + * Fire an event signifying a blackboard post by an ingest module. + * + * @param moduleDataEvent A ModuleDataEvent with the details of the posting. + */ + void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { + fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.DATA, moduleDataEvent, null)); + } + + /** + * Fire an event signifying discovery of additional content by an ingest + * module. + * + * @param moduleDataEvent A ModuleContentEvent with the details of the new + * content. + */ + void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { + fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.CONTENT_CHANGED, moduleContentEvent, null)); + } + + /** + * Post a message to the ingest messages in box. + * + * @param message The message to be posted. + */ void postIngestMessage(IngestMessage message) { if (ingestMessageBox != null) { ingestMessageBox.displayMessage(message); @@ -262,11 +338,10 @@ public class IngestManager { } /** - * Get free disk space of a drive where ingest data are written to That - * drive is being monitored by IngestMonitor thread when ingest is running. - * Use this method to get amount of free disk space anytime. + * Get the free disk space of the drive where to which ingest data is being + * written, as reported by the ingest monitor. * - * @return amount of disk space, -1 if unknown + * @return Free disk space, -1 if unknown // RJCTODO: What units? */ long getFreeDiskSpace() { if (ingestMonitor != null) { @@ -276,33 +351,20 @@ public class IngestManager { } } - private void reportRunIngestModulesTaskDone(long taskId) { - ingestTasks.remove(taskId); + /** + * A Runnable that creates ingest jobs and submits the initial data source + * and file ingest tasks to the task schedulers. + */ + private class StartIngestJobsThread implements Runnable { - List completedJobs = new ArrayList<>(); - for (IngestJob job : ingestJobs.values()) { - job.releaseIngestPipelinesForThread(taskId); - if (job.areIngestPipelinesShutDown() == true) { - completedJobs.add(job.getId()); - } - } - - for (Long jobId : completedJobs) { - IngestJob job = ingestJobs.remove(jobId); - fireEventTasksExecutor.submit(new FireIngestJobEventTask(jobId, job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED : IngestEvent.INGEST_JOB_COMPLETED)); - } - } - - private class StartIngestJobsTask implements Runnable { - - private final long id; + private final long threadId; private final List dataSources; private final List moduleTemplates; private final boolean processUnallocatedSpace; private ProgressHandle progress; - StartIngestJobsTask(long taskId, List dataSources, List moduleTemplates, boolean processUnallocatedSpace) { - this.id = taskId; + StartIngestJobsThread(long threadId, List dataSources, List moduleTemplates, boolean processUnallocatedSpace) { + this.threadId = threadId; this.dataSources = dataSources; this.moduleTemplates = moduleTemplates; this.processUnallocatedSpace = processUnallocatedSpace; @@ -321,12 +383,16 @@ public class IngestManager { "IngestManager.StartIngestJobsTask.run.cancelling", displayName)); } - IngestManager.getInstance().cancelIngestJobs(); + cancelFileIngestThread(threadId); return true; } }); + progress.start(dataSources.size() * 2); + + if (!ingestMonitor.isRunning()) { + ingestMonitor.start(); + } - progress.start(2 * dataSources.size()); int workUnitsCompleted = 0; for (Content dataSource : dataSources) { if (Thread.currentThread().isInterrupted()) { @@ -334,14 +400,7 @@ public class IngestManager { } // Create an ingest job. - IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace); - ingestJobs.put(ingestJob.getId(), ingestJob); - - // Start at least one instance of each kind of ingest - // pipeline for this ingest job. This allows for an early out - // if the full ingest module lineup specified by the user - // cannot be started up. - List errors = ingestJob.startUpIngestPipelines(); + List errors = IngestJob.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace); if (!errors.isEmpty()) { // Report the error to the user. StringBuilder moduleStartUpErrors = new StringBuilder(); @@ -367,169 +426,116 @@ public class IngestManager { JOptionPane.showMessageDialog(null, notifyMessage.toString(), NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); - - // Jettison the ingest job and move on to the next one. - ingestJob.cancel(); - ingestJobs.remove(ingestJob.getId()); - break; } - // Queue the data source ingest tasks for the ingest job. + fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED)); + + // Queue a data source ingest task for the ingest job. final String inputName = dataSource.getName(); progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg1", + NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg1", inputName), workUnitsCompleted); - scheduler.getDataSourceIngestScheduler().queueForIngest(ingestJob); + DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(ingestJob, ingestJob.getDataSource())); progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg2", + NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg2", inputName), ++workUnitsCompleted); // Queue the file ingest tasks for the ingest job. progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg3", + NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg3", inputName), workUnitsCompleted); - scheduler.getFileIngestScheduler().queueForIngest(ingestJob); + FileIngestTaskScheduler.getInstance().addTasks(ingestJob, ingestJob.getDataSource()); progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg4", + NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg4", inputName), ++workUnitsCompleted); if (!Thread.currentThread().isInterrupted()) { - if (!ingestMonitor.isRunning()) { - ingestMonitor.start(); - } - - long taskId = ingestTaskId.incrementAndGet(); - Future task = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId)); - ingestTasks.put(taskId, task); - - int numberOfFileTasksRequested = getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileTasksRequested; ++i) { - taskId = ingestTaskId.incrementAndGet(); - task = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId)); - ingestTasks.put(taskId, task); - } - - fireEventTasksExecutor.submit(new FireIngestJobEventTask(ingestJob.getId(), IngestEvent.INGEST_JOB_STARTED)); + break; } } } catch (Exception ex) { - String message = String.format("StartIngestJobsTask (id=%d) caught exception", id); //NON-NLS + String message = String.format("StartIngestJobsTask (id=%d) caught exception", threadId); //NON-NLS logger.log(Level.SEVERE, message, ex); MessageNotifyUtil.Message.error( NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg")); } finally { progress.finish(); - ingestTasks.remove(id); + startIngestJobThreads.remove(threadId); } } } - private class RunDataSourceIngestModulesTask implements Runnable { - - private final long id; - - RunDataSourceIngestModulesTask(long taskId) { - id = taskId; - } + /** + * A Runnable that acts as a consumer for the data ingest task scheduler's + * task queue. + */ + private class DataSourceIngestThread implements Runnable { @Override public void run() { - try { - IngestScheduler.DataSourceIngestScheduler scheduler = IngestScheduler.getInstance().getDataSourceIngestScheduler(); - IngestJob job = scheduler.getNextTask(); - while (job != null) { - if (Thread.currentThread().isInterrupted()) { - break; - } - job.getDataSourceIngestPipelineForThread(id).process(); - job = scheduler.getNextTask(); + DataSourceIngestTaskScheduler scheduler = DataSourceIngestTaskScheduler.getInstance(); + while (true) { + try { + DataSourceIngestTask task = scheduler.getNextTask(); // Blocks. + task.execute(); + } catch (InterruptedException ex) { + break; + } + if (Thread.currentThread().isInterrupted()) { + break; } - } catch (Exception ex) { - String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS - logger.log(Level.SEVERE, message, ex); - } finally { - reportRunIngestModulesTaskDone(id); } } } - private class RunFileSourceIngestModulesTask implements Runnable { - - private final long id; - - RunFileSourceIngestModulesTask(long taskId) { - id = taskId; - } + /** + * A Runnable that acts as a consumer for the file task scheduler's task + * queue. + */ + private static class FileIngestThread implements Runnable { @Override public void run() { - try { - IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); - FileIngestTask task = fileScheduler.getNextTask(); - while (task != null) { - if (Thread.currentThread().isInterrupted()) { - break; - } - IngestJob job = task.getJob(); - job.updateFileTasksProgressBar(task.getFile().getName()); - job.getFileIngestPipelineForThread(id).process(task.getFile()); - task = fileScheduler.getNextTask(); + FileIngestTaskScheduler scheduler = FileIngestTaskScheduler.getInstance(); + while (true) { + try { + FileIngestTask task = scheduler.getNextTask(); // Blocks. + task.execute(); + } catch (InterruptedException ex) { + break; + } + if (Thread.currentThread().isInterrupted()) { + break; } - } catch (Exception ex) { - String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS - logger.log(Level.SEVERE, message, ex); - } finally { - reportRunIngestModulesTaskDone(id); } } } - private class FireIngestJobEventTask implements Runnable { + /** + * A Runnable that fire ingest events to ingest manager property change + * listeners. + */ + private class FireIngestEventThread implements Runnable { - private final long ingestJobId; private final IngestEvent event; + private final Object oldValue; + private final Object newValue; - FireIngestJobEventTask(long ingestJobId, IngestEvent event) { - this.ingestJobId = ingestJobId; + FireIngestEventThread(IngestEvent event, Object oldValue, Object newValue) { this.event = event; + this.oldValue = oldValue; + this.newValue = newValue; } @Override public void run() { - fireIngestJobEvent(event.toString(), ingestJobId); - } - } - - private class IngestCancellationWorker extends SwingWorker { - - @Override - protected Void doInBackground() throws Exception { - // First mark all of the ingest jobs as cancelled. This way the - // ingest modules will know they are being shut down due to - // cancellation when the cancelled run ingest module tasks release - // their pipelines. - for (IngestJob job : ingestJobs.values()) { - job.cancel(); - } - - for (Future task : ingestTasks.values()) { - task.cancel(true); - } - - // Jettision the remaining data source and file ingest tasks. - scheduler.getFileIngestScheduler().emptyQueues(); - scheduler.getDataSourceIngestScheduler().emptyQueues(); - - return null; - } - - @Override - protected void done() { try { - super.get(); - } catch (CancellationException | InterruptedException ex) { - } catch (Exception ex) { - logger.log(Level.SEVERE, "Error while cancelling ingest jobs", ex); //NON-NLS + pcs.firePropertyChange(event.toString(), oldValue, newValue); + } catch (Exception e) { + logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS + MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), // RJCTODO: Oddly named strings + NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), + MessageNotifyUtil.MessageType.ERROR); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java index 7b2bcd9804..4a6f63caff 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java @@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content; manager = IngestManager.getInstance(); } try { - manager.cancelIngestJobs(); + manager.cancelAllIngestJobs(); } finally { //clear inbox clearMessages(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index 17ef4f41fc..3fee18f391 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -165,7 +165,7 @@ public final class IngestMonitor { final String diskPath = root.getAbsolutePath(); MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS - manager.cancelIngestJobs(); + manager.cancelAllIngestJobs(); IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage( NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath), NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath))); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java deleted file mode 100644 index 70abc43381..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ /dev/null @@ -1,744 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2012-2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.ContentVisitor; -import org.sleuthkit.datamodel.DerivedFile; -import org.sleuthkit.datamodel.Directory; -import org.sleuthkit.datamodel.File; -import org.sleuthkit.datamodel.FileSystem; -import org.sleuthkit.datamodel.VirtualDirectory; -import org.sleuthkit.datamodel.LayoutFile; -import org.sleuthkit.datamodel.LocalFile; -import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM; -import org.sleuthkit.datamodel.TskData.TSK_FS_META_TYPE_ENUM; - -/** - * Enqueues data source ingest and file ingest tasks for processing. - */ -final class IngestScheduler { - - private static IngestScheduler instance; - private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName()); - private final DataSourceIngestScheduler dataSourceIngestScheduler = new DataSourceIngestScheduler(); - private final FileIngestScheduler fileIngestScheduler = new FileIngestScheduler(); - - private IngestScheduler() { - } - - static synchronized IngestScheduler getInstance() { - if (instance == null) { - instance = new IngestScheduler(); - } - - return instance; - } - - DataSourceIngestScheduler getDataSourceIngestScheduler() { - return dataSourceIngestScheduler; - } - - FileIngestScheduler getFileIngestScheduler() { - return fileIngestScheduler; - } - - static class FileIngestScheduler { - - private TreeSet rootDirectoryTasks; - private List directoryTasks; - private LinkedList fileTasks; //need to add to start and end quickly - private int filesEnqueuedEst = 0; - private int filesDequeued = 0; - private final static int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() - | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() - | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() - | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); - - private FileIngestScheduler() { - rootDirectoryTasks = new TreeSet<>(new RootTaskComparator()); - directoryTasks = new ArrayList<>(); - fileTasks = new LinkedList<>(); - resetCounters(); - } - - private void resetCounters() { - filesEnqueuedEst = 0; - filesDequeued = 0; - } - - @Override - public synchronized String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.rootDirs.text")).append(rootDirectoryTasks.size()); - for (FileIngestTask task : rootDirectoryTasks) { - sb.append(task.toString()).append(" "); - } - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curDirs.text")).append(directoryTasks.size()); - for (FileIngestTask task : directoryTasks) { - sb.append(task.toString()).append(" "); - } - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curFiles.text")).append(fileTasks.size()); - for (FileIngestTask task : fileTasks) { - sb.append(task.toString()).append(" "); - } - return sb.toString(); - } - - synchronized void queueForIngest(IngestJob dataSourceTask) { - Content dataSource = dataSourceTask.getDataSource(); - Collection rootObjects = dataSource.accept(new GetRootDirVisitor()); - List firstLevelFiles = new ArrayList<>(); - if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { - // The data source is file. - firstLevelFiles.add((AbstractFile) dataSource); - } else { - for (AbstractFile root : rootObjects) { - List children; - try { - children = root.getChildren(); - if (children.isEmpty()) { - //add the root itself, could be unalloc file, child of volume or image - firstLevelFiles.add(root); - } else { - //root for fs root dir, schedule children dirs/files - for (Content child : children) { - if (child instanceof AbstractFile) { - firstLevelFiles.add((AbstractFile) child); - } - } - } - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS - } - } - } - - for (AbstractFile firstLevelFile : firstLevelFiles) { - FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask); - if (shouldEnqueueTask(fileTask)) { - rootDirectoryTasks.add(fileTask); - } - } - - // Update approx count of files to process in queues - filesEnqueuedEst = queryNumFilesinEnqueuedContents(); - - // Reshuffle/update the dir and file level queues if needed - updateQueues(); - } - - synchronized void queueFile(IngestJob ingestJob, AbstractFile file) { - FileIngestTask fileTask = new FileIngestTask(file, ingestJob); - if (shouldEnqueueTask(fileTask)) { - fileTasks.addFirst(fileTask); - ++filesEnqueuedEst; - } - } - - float getPercentageDone() { - if (filesEnqueuedEst == 0) { - return 0; - } - return ((100.f) * filesDequeued) / filesEnqueuedEst; - } - - /** - * query num files enqueued total num of files to be enqueued. - * - * Counts all files for all the sources currently in the queues. - * - * @return approx. total num of files enqueued (or to be enqueued) - */ - private synchronized int queryNumFilesinEnqueuedContents() { - int totalFiles = 0; - List contents = this.getSourceContent(); - - final GetFilesCountVisitor countVisitor = - new GetFilesCountVisitor(); - for (Content content : contents) { - totalFiles += content.accept(countVisitor); - } - - logger.log(Level.INFO, "Total files to queue up: {0}", totalFiles); //NON-NLS - - return totalFiles; - } - - /** - * get total est. number of files to be enqueued for current ingest - * input sources in queues - * - * @return total number of files - */ - int getFilesEnqueuedEst() { - return filesEnqueuedEst; - } - - /** - * Get number of files dequeued so far. This is reset after the same - * content is enqueued that is already in a queue - * - * @return number of files dequeued so far - */ - int getFilesDequeued() { - return filesDequeued; - } - - synchronized FileIngestTask getNextTask() { - final FileIngestTask task = fileTasks.pollLast(); - if (task != null) { - filesDequeued++; - updateQueues(); - } - return task; - } - - /** - * Shuffle the queues so that there are files in the files queue. - * - * @returns true if no more data in queue - */ - private synchronized void updateQueues() { - - // we loop because we could have a directory that has all files - // that do not get enqueued - while (true) { - // There are files in the queue, we're done - if (this.fileTasks.isEmpty() == false) { - return; - } - - // fill in the directory queue if it is empty. - if (this.directoryTasks.isEmpty()) { - // bail out if root is also empty -- we are done - if (rootDirectoryTasks.isEmpty()) { - return; - } - FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst(); - directoryTasks.add(rootTask); - } - - //pop and push AbstractFile directory children if any - //add the popped and its leaf children onto cur file list - FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); - final AbstractFile parentFile = parentTask.file; - - // add itself to the file list - if (shouldEnqueueTask(parentTask)) { - this.fileTasks.addLast(parentTask); - } - - // add its children to the file and directory lists - try { - List children = parentFile.getChildren(); - for (Content c : children) { - if (c instanceof AbstractFile) { - AbstractFile childFile = (AbstractFile) c; - FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getJob()); - - if (childFile.hasChildren()) { - this.directoryTasks.add(childTask); - } else if (shouldEnqueueTask(childTask)) { - this.fileTasks.addLast(childTask); - } - } - } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Could not get children of file and update file queues: " //NON-NLS - + parentFile.getName(), ex); - } - } - } - - /** - * Return list of content objects that are in the queue to be processed. - * - * Helpful to determine whether ingest for particular input Content is - * active - * - * @return list of parent source content objects for files currently - * enqueued - */ - synchronized List getSourceContent() { - final Set contentSet = new HashSet<>(); - - for (FileIngestTask task : rootDirectoryTasks) { - contentSet.add(task.getJob().getDataSource()); - } - for (FileIngestTask task : directoryTasks) { - contentSet.add(task.getJob().getDataSource()); - } - for (FileIngestTask task : fileTasks) { - contentSet.add(task.getJob().getDataSource()); - } - - return new ArrayList<>(contentSet); - } - - synchronized void emptyQueues() { - this.rootDirectoryTasks.clear(); - this.directoryTasks.clear(); - this.fileTasks.clear(); - } - - /** - * Check if the file is a special file that we should skip - * - * @param processTask a task whose file to check if should be queued of - * skipped - * @return true if should be enqueued, false otherwise - */ - private static boolean shouldEnqueueTask(final FileIngestTask processTask) { - final AbstractFile aFile = processTask.file; - - //if it's unalloc file, skip if so scheduled - if (processTask.getJob().shouldProcessUnallocatedSpace() == false - && aFile.getType().equals(TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS //unalloc files - )) { - return false; - } - - String fileName = aFile.getName(); - if (fileName.equals(".") || fileName.equals("..")) { - return false; - } else if (aFile instanceof org.sleuthkit.datamodel.File) { - final org.sleuthkit.datamodel.File f = (File) aFile; - - //skip files in root dir, starting with $, containing : (not default attributes) - //with meta address < 32, i.e. some special large NTFS and FAT files - FileSystem fs = null; - try { - fs = f.getFileSystem(); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS - } - TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP; - if (fs != null) { - fsType = fs.getFsType(); - } - - if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { - //not fat or ntfs, accept all files - return true; - } - - boolean isInRootDir = false; - try { - isInRootDir = f.getParentDirectory().isRoot(); - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS - } - - if (isInRootDir && f.getMetaAddr() < 32) { - String name = f.getName(); - - if (name.length() > 0 - && name.charAt(0) == '$' - && name.contains(":")) { - return false; - } - } else { - return true; - } - - } - - return true; - } - - static class FileIngestTask { - - private final AbstractFile file; - private final IngestJob task; - - private FileIngestTask(AbstractFile file, IngestJob task) { - this.file = file; - this.task = task; - } - - public IngestJob getJob() { - return task; - } - - public AbstractFile getFile() { - return file; - } - - @Override - public String toString() { - try { - return "ProcessTask{" + "file=" + file.getId() + ": " //NON-NLS - + file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}'; - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS - } - return "ProcessTask{" + "file=" + file.getId() + ": " //NON-NLS - + file.getName() + '}'; - } - - /** - * two process tasks are equal when the file/dir and modules are the - * same this enables are not to queue up the same file/dir, modules - * tuples into the root dir set - * - * @param obj - * @return - */ - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final FileIngestTask other = (FileIngestTask) obj; - if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { - return false; - } - IngestJob thisTask = this.getJob(); - IngestJob otherTask = other.getJob(); - - if (thisTask != otherTask - && (thisTask == null || !thisTask.equals(otherTask))) { - return false; - } - return true; - } - - @Override - public int hashCode() { - int hash = 5; - hash = 47 * hash + Objects.hashCode(this.file); - hash = 47 * hash + Objects.hashCode(this.task); - return hash; - } - } - - /** - * Root dir sorter - */ - private static class RootTaskComparator implements Comparator { - - @Override - public int compare(FileIngestTask q1, FileIngestTask q2) { - AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file); - AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file); - if (p1 == p2) { - return (int) (q2.file.getId() - q1.file.getId()); - } else { - return p2.ordinal() - p1.ordinal(); - } - - } - - /** - * Priority determination for sorted AbstractFile, used by - * RootDirComparator - */ - private static class AbstractFilePriority { - - enum Priority { - - LAST, LOW, MEDIUM, HIGH - }; - static final List LAST_PRI_PATHS = new ArrayList<>(); - static final List LOW_PRI_PATHS = new ArrayList<>(); - static final List MEDIUM_PRI_PATHS = new ArrayList<>(); - static final List HIGH_PRI_PATHS = new ArrayList<>(); - - /* prioritize root directory folders based on the assumption that we are - * looking for user content. Other types of investigations may want different - * priorities. */ - static { - // these files have no structure, so they go last - //unalloc files are handled as virtual files in getPriority() - //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); - //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE)); - LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE)); - LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE)); - - // orphan files are often corrupt and windows does not typically have - // user content, so put them towards the bottom - LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE)); - LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE)); - - // all other files go into the medium category too - MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE)); - - // user content is top priority - HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE)); - HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE)); - HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE)); - HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE)); - } - - /** - * Get the scheduling priority for a given file. - * - * @param abstractFile - * @return - */ - static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) { - if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { - //quickly filter out unstructured content - //non-fs virtual files and dirs, such as representing unalloc space - return AbstractFilePriority.Priority.LAST; - } - - //determine the fs files priority by name - final String path = abstractFile.getName(); - - if (path == null) { - return AbstractFilePriority.Priority.MEDIUM; - } - - for (Pattern p : HIGH_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.HIGH; - } - } - - for (Pattern p : MEDIUM_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.MEDIUM; - } - } - - for (Pattern p : LOW_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.LOW; - } - } - - for (Pattern p : LAST_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.LAST; - } - } - - //default is medium - return AbstractFilePriority.Priority.MEDIUM; - } - } - } - - /** - * Get counts of ingestable files/dirs for the content input source. - * - * Note, also includes counts of all unalloc children files (for the fs, - * image, volume) even if ingest didn't ask for them - */ - static class GetFilesCountVisitor extends ContentVisitor.Default { - - @Override - public Long visit(FileSystem fs) { - //recursion stop here - //case of a real fs, query all files for it - - SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase(); - - StringBuilder queryB = new StringBuilder(); - queryB.append("( (fs_obj_id = ").append(fs.getId()); //NON-NLS - //queryB.append(") OR (fs_obj_id = NULL) )"); - queryB.append(") )"); - queryB.append(" AND ( (meta_type = ").append(TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS - queryB.append(") OR (meta_type = ").append(TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS - queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS - queryB.append(") )"); - - //queryB.append( "AND (type = "); - //queryB.append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.getFileType()); - //queryB.append(")"); - try { - final String query = queryB.toString(); - logger.log(Level.INFO, "Executing count files query: {0}", query); //NON-NLS - return sc.countFilesWhere(query); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); //NON-NLS - return 0L; - } - } - - @Override - public Long visit(LayoutFile lf) { - //recursion stop here - //case of LayoutFile child of Image or Volume - return 1L; - } - - private long getCountFromChildren(Content content) { - long count = 0; - try { - List children = content.getChildren(); - if (children.size() > 0) { - for (Content child : children) { - count += child.accept(this); - } - } else { - count = 1; - } - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not get count of objects from children to get num of total files to be ingested", ex); //NON-NLS - } - return count; - } - - @Override - protected Long defaultVisit(Content cntnt) { - //recurse assuming this is image/vs/volume - //recursion stops at fs or unalloc file - return getCountFromChildren(cntnt); - } - } - - /** - * Visitor that gets a collection of top level objects to be scheduled, - * such as root Dirs (if there is FS) or LayoutFiles and virtual - * directories, also if there is no FS. - */ - static class GetRootDirVisitor extends GetFilesContentVisitor { - - @Override - public Collection visit(VirtualDirectory ld) { - //case when we hit a layout directoryor local file container, not under a real FS - //or when root virt dir is scheduled - Collection ret = new ArrayList<>(); - ret.add(ld); - return ret; - } - - @Override - public Collection visit(LayoutFile lf) { - //case when we hit a layout file, not under a real FS - Collection ret = new ArrayList<>(); - ret.add(lf); - return ret; - } - - @Override - public Collection visit(Directory drctr) { - //we hit a real directory, a child of real FS - - Collection ret = new ArrayList<>(); - - ret.add(drctr); - - return ret; - } - - @Override - public Collection visit(FileSystem fs) { - return getAllFromChildren(fs); - - } - - @Override - public Collection visit(File file) { - //can have derived files - return getAllFromChildren(file); - } - - @Override - public Collection visit(DerivedFile derivedFile) { - //can have derived files - //TODO test this and overall scheduler with derived files - return getAllFromChildren(derivedFile); - } - - @Override - public Collection visit(LocalFile localFile) { - //can have local files - //TODO test this and overall scheduler with local files - return getAllFromChildren(localFile); - } - } - } - - static class DataSourceIngestScheduler { - - private final LinkedList tasks = new LinkedList<>(); - - private DataSourceIngestScheduler() { - } - - synchronized void queueForIngest(IngestJob job) { - try { - if (job.getDataSource().getParent() != null) { - logger.log(Level.SEVERE, "Only parent-less Content (data sources) can be scheduled for DataSource ingest, skipping: {0}", job.getDataSource()); //NON-NLS - return; - } - } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error validating data source to be scheduled for DataSource ingest" + job.getDataSource(), e); //NON-NLS - return; - } - - tasks.addLast(job); - } - - public synchronized IngestJob getNextTask() { - return tasks.pollFirst(); - } - - synchronized void emptyQueues() { - tasks.clear(); - } - - synchronized int getCount() { - return tasks.size(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.DataSourceScheduler.toString.size")) - .append(getCount()); - for (IngestJob task : tasks) { - sb.append(task.toString()).append(" "); - } - return sb.toString(); - } - } -} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java index 8efe079e1d..c573753281 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java @@ -95,7 +95,7 @@ public final class IngestServices { * artifact data */ public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { - IngestManager.fireModuleDataEvent(moduleDataEvent); + IngestManager.fireIngestModuleDataEvent(moduleDataEvent); } /** @@ -107,7 +107,7 @@ public final class IngestServices { * changed */ public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { - IngestManager.fireModuleContentEvent(moduleContentEvent); + IngestManager.fireIngestModuleContentEvent(moduleContentEvent); } /** From a77d566a82cc7b8c97f7dfd2f15b45baadee43e9 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 1 May 2014 09:41:01 -0400 Subject: [PATCH 2/5] Preserve intermediate state of improved ingest framework --- .../DirectoryTreeTopComponent.java | 3 +- .../autopsy/ingest/Bundle.properties | 8 +- .../autopsy/ingest/Bundle_ja.properties | 4 - .../DataSourceIngestModuleProgress.java | 14 +- .../ingest/DataSourceIngestPipeline.java | 18 +- .../autopsy/ingest/DataSourceIngestTask.java | 4 +- .../ingest/DataSourceIngestTaskScheduler.java | 38 ++-- .../autopsy/ingest/FileIngestPipeline.java | 17 +- .../autopsy/ingest/FileIngestTask.java | 46 ++--- .../ingest/FileIngestTaskScheduler.java | 124 +++++------ .../sleuthkit/autopsy/ingest/IngestJob.java | 195 +++++++++--------- .../autopsy/ingest/IngestJobContext.java | 4 +- .../autopsy/ingest/IngestManager.java | 173 +++++++++------- .../autopsy/ingest/IngestServices.java | 4 +- .../hashdatabase/HashLookupSettingsPanel.java | 2 +- .../KeywordSearchEditListPanel.java | 2 +- .../KeywordSearchListsViewerPanel.java | 2 +- c:casesSamll2Againautopsy.db | 0 18 files changed, 329 insertions(+), 329 deletions(-) create mode 100755 c:casesSamll2Againautopsy.db diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index a03933abf6..f7067adf6e 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -131,7 +131,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private void setListener() { Case.addPropertyChangeListener(this);// add this class to listen to any changes in the Case.java class this.em.addPropertyChangeListener(this); - IngestManager.addPropertyChangeListener(this); + IngestManager.getInstance().addIngestJobEventListener(this); + IngestManager.getInstance().addIngestModuleEventListener(this); } public void setDirectoryListingActive() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 144c7359fd..253f4f390d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -60,13 +60,9 @@ IngestMessagePanel.sortByComboBox.model.priority=Priority IngestMessagesToolbar.customizeButton.toolTipText=Ingest Messages IngestMessageTopComponent.initComponents.name=Ingest Inbox IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=Unable to start up one or more ingest modules, ingest job cancelled. -IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=Please disable the failed modules or fix the errors and then restart ingest\ +IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=Please disable the failed modules or fix the errors and then restart ingest \ by right clicking on the data source and selecting Run Ingest Modules. -IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\:\ +IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\: \ \ {0} IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=Ingest Failure -IngestManager.StartIngestJobsTask.run.progress.msg1=Data source ingest tasks for {0} -IngestManager.StartIngestJobsTask.run.progress.msg2=Data source ingest tasks for {0} -IngestManager.StartIngestJobsTask.run.progress.msg3=Data source ingest tasks for {0} -IngestManager.StartIngestJobsTask.run.progress.msg4=Data source ingest tasks for {0} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties index e3433daff3..c6a50ca915 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties @@ -56,10 +56,6 @@ IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7 IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3 IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30b5\u30a4\u30ba\uff1a OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8 -IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af -IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af -IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af -IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30a8\u30e9\u30fc\uff1a\ \ {0} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index 39fbca98fe..db2c4b3e37 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -18,16 +18,18 @@ */ package org.sleuthkit.autopsy.ingest; +import org.netbeans.api.progress.ProgressHandle; + /** * Used by data source ingest modules to report progress. */ public class DataSourceIngestModuleProgress { - private final IngestJob ingestJob; + private final ProgressHandle progress; private final String moduleDisplayName; - DataSourceIngestModuleProgress(IngestJob ingestJob, String moduleDisplayName) { - this.ingestJob = ingestJob; + DataSourceIngestModuleProgress(ProgressHandle progress, String moduleDisplayName) { + this.progress = progress; this.moduleDisplayName = moduleDisplayName; } @@ -40,7 +42,7 @@ public class DataSourceIngestModuleProgress { * data source. */ public void switchToDeterminate(int workUnits) { - ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits); + progress.switchToDeterminate(workUnits); } /** @@ -48,7 +50,7 @@ public class DataSourceIngestModuleProgress { * the total work units to process the data source is unknown. */ public void switchToIndeterminate() { - ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate(); + progress.switchToIndeterminate(); } /** @@ -58,6 +60,6 @@ public class DataSourceIngestModuleProgress { * @param workUnits Number of work units performed so far by the module. */ public void progress(int workUnits) { - ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits); + progress.progress(this.moduleDisplayName, workUnits); } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index b824ffe615..86d001f1f8 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.netbeans.api.progress.ProgressHandle; import org.sleuthkit.datamodel.Content; /** @@ -30,12 +31,12 @@ import org.sleuthkit.datamodel.Content; */ final class DataSourceIngestPipeline { - private final IngestJob job; + private final IngestJobContext context; private final List moduleTemplates; private List modules = new ArrayList<>(); - DataSourceIngestPipeline(IngestJob job, List moduleTemplates) { - this.job = job; + DataSourceIngestPipeline(IngestJobContext context, List moduleTemplates) { + this.context = context; this.moduleTemplates = moduleTemplates; } @@ -50,7 +51,6 @@ final class DataSourceIngestPipeline { for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); - IngestJobContext context = new IngestJobContext(job); try { module.startUp(context); modulesByClass.put(module.getClassName(), module); @@ -75,26 +75,26 @@ final class DataSourceIngestPipeline { return errors; } - List process() { + List process(Content dataSource, ProgressHandle progress) { List errors = new ArrayList<>(); for (DataSourceIngestModuleDecorator module : this.modules) { try { - module.process(job.getDataSource(), new DataSourceIngestModuleProgress(job, module.getDisplayName())); + module.process(dataSource, new DataSourceIngestModuleProgress(progress, module.getDisplayName())); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (job.isCancelled()) { + if (context.isJobCancelled()) { break; } } return errors; } - List shutDown(boolean ingestJobCancelled) { + List shutDown() { List errors = new ArrayList<>(); for (DataSourceIngestModuleDecorator module : this.modules) { try { - module.shutDown(ingestJobCancelled); + module.shutDown(context.isJobCancelled()); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index af27d9b8c0..2ad3c644e3 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -38,7 +38,7 @@ final class DataSourceIngestTask { return dataSource; } - void execute() { - ingestJob.process(); + void execute() throws InterruptedException { + ingestJob.process(dataSource); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java index 22a6124396..ca22a6267b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java @@ -32,26 +32,30 @@ final class DataSourceIngestTaskScheduler { private DataSourceIngestTaskScheduler() { } - synchronized void addTask(DataSourceIngestTask task) throws InterruptedException { - task.getIngestJob().notifyTaskPending(); - try { - tasks.put(task); - } - catch (InterruptedException ex) { - // RJCTOD: Need a safety notification to undo above + synchronized void addTask(DataSourceIngestTask task) { + // The capacity of the tasks queue is not bounded, so the call + // to put() should not block except for normal synchronized access. + // Still, notify the job that the task has been added first so that + // the take() of the task cannot occur before the notification. + task.getIngestJob().notifyTaskAdded(); + + // If the thread executing this code is ever interrupted, it is + // because the number of ingest threads has been decreased while + // ingest jobs are running. This thread will exit in an orderly fashion, + // but the task still needs to be enqueued rather than lost. + while (true) { + try { + tasks.put(task); + break; + } catch (InterruptedException ex) { + // Reset the interrupted status of the thread so the orderly + // exit can occur in the intended place. + Thread.currentThread().interrupt(); + } } } DataSourceIngestTask getNextTask() throws InterruptedException { - return tasks.take(); - } - - boolean hasTasksForIngestJob(long jobId) { - for (DataSourceIngestTask task : tasks) { - if (task.getIngestJobId() == jobId) { - return true; - } - } - return false; + return tasks.take(); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 2cd7d838e4..1b3743e616 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -30,12 +30,12 @@ import org.sleuthkit.datamodel.AbstractFile; */ final class FileIngestPipeline { - private final IngestJob job; + private final IngestJobContext context; private final List moduleTemplates; private List modules = new ArrayList<>(); - FileIngestPipeline(IngestJob task, List moduleTemplates) { - this.job = task; + FileIngestPipeline(IngestJobContext context, List moduleTemplates) { + this.context = context; this.moduleTemplates = moduleTemplates; } @@ -50,7 +50,6 @@ final class FileIngestPipeline { for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); - IngestJobContext context = new IngestJobContext(job); try { module.startUp(context); modulesByClass.put(module.getClassName(), module); @@ -83,22 +82,22 @@ final class FileIngestPipeline { } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (job.isCancelled()) { + if (context.isJobCancelled()) { break; } } file.close(); - if (!job.isCancelled()) { - IngestManager.fireFileIngestDone(file.getId()); + if (!context.isJobCancelled()) { + IngestManager.getInstance().fireFileIngestDone(file.getId()); } return errors; } - List shutDown(boolean ingestJobCancelled) { + List shutDown() { List errors = new ArrayList<>(); for (FileIngestModuleDecorator module : this.modules) { try { - module.shutDown(ingestJobCancelled); + module.shutDown(context.isJobCancelled()); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index db0edf9edf..54ed9a8501 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -19,17 +19,16 @@ package org.sleuthkit.autopsy.ingest; import java.util.Objects; -import java.util.logging.Level; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; final class FileIngestTask { - final AbstractFile file; - private final IngestJob ingestJob; - FileIngestTask(AbstractFile file, IngestJob task) { - this.file = file; + private final IngestJob ingestJob; + private final AbstractFile file; + + FileIngestTask(IngestJob task, AbstractFile file) { this.ingestJob = task; + this.file = file; } public IngestJob getIngestJob() { @@ -39,30 +38,11 @@ final class FileIngestTask { public AbstractFile getFile() { return file; } - - void execute(long threadId) { + + void execute() throws InterruptedException { ingestJob.process(file); } - @Override - public String toString() { //RJCTODO: May not keep this - try { - return "ProcessTask{" + "file=" + file.getId() + ": " + file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}'; - } catch (TskCoreException ex) { - // RJCTODO -// FileIngestTaskScheduler.logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS - } - return "ProcessTask{" + "file=" + file.getId() + ": " + file.getName() + '}'; - } - - /** - * two process tasks are equal when the file/dir and modules are the - * same this enables are not to queue up the same file/dir, modules - * tuples into the root dir set - * - * @param obj - * @return - */ @Override public boolean equals(Object obj) { if (obj == null) { @@ -71,13 +51,11 @@ final class FileIngestTask { if (getClass() != obj.getClass()) { return false; } - final FileIngestTask other = (FileIngestTask) obj; - if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { + FileIngestTask other = (FileIngestTask) obj; + if (this.ingestJob != other.ingestJob && (this.ingestJob == null || !this.ingestJob.equals(other.ingestJob))) { return false; } - IngestJob thisTask = this.getIngestJob(); - IngestJob otherTask = other.getIngestJob(); - if (thisTask != otherTask && (thisTask == null || !thisTask.equals(otherTask))) { + if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { return false; } return true; @@ -86,8 +64,8 @@ final class FileIngestTask { @Override public int hashCode() { int hash = 5; - hash = 47 * hash + Objects.hashCode(this.file); hash = 47 * hash + Objects.hashCode(this.ingestJob); + hash = 47 * hash + Objects.hashCode(this.file); return hash; - } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java index 280012ebf8..fcbceab6ec 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java @@ -43,23 +43,20 @@ import org.sleuthkit.datamodel.VirtualDirectory; final class FileIngestTaskScheduler { private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); - private static FileIngestTaskScheduler instance; + private static FileIngestTaskScheduler instance = new FileIngestTaskScheduler(); private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); private final List directoryTasks = new ArrayList<>(); - private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); // Unlimited capacity + private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); - static synchronized FileIngestTaskScheduler getInstance() { - if (instance == null) { - instance = new FileIngestTaskScheduler(); - } + static FileIngestTaskScheduler getInstance() { return instance; } private FileIngestTaskScheduler() { } - synchronized void addTasks(IngestJob dataSourceTask, Content dataSource) { + synchronized void addTasks(IngestJob job, Content dataSource) throws InterruptedException { Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); List firstLevelFiles = new ArrayList<>(); if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { @@ -87,9 +84,9 @@ final class FileIngestTaskScheduler { } } for (AbstractFile firstLevelFile : firstLevelFiles) { - FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask); + FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); if (shouldEnqueueTask(fileTask)) { - rootDirectoryTasks.add(fileTask); + addTaskToRootDirectoryQueue(fileTask); } } @@ -97,16 +94,9 @@ final class FileIngestTaskScheduler { updateQueues(); } - synchronized void addTask(IngestJob ingestJob, AbstractFile file) { - try { - FileIngestTask fileTask = new FileIngestTask(file, ingestJob); - if (shouldEnqueueTask(fileTask)) { - fileTask.getIngestJob().notifyTaskPending(); - fileTasks.put(fileTask); // Queue has unlimited capacity, does not block. - } - } catch (InterruptedException ex) { - // RJCTODO: Perhaps this is the convenience method? - // RJCTODO: Need undo + synchronized void addTask(FileIngestTask task) { + if (shouldEnqueueTask(task)) { + addTaskToFileQueue(task); } } @@ -116,29 +106,7 @@ final class FileIngestTaskScheduler { return task; } - synchronized boolean hasTasksForJob(long ingestJobId) { - for (FileIngestTask task : rootDirectoryTasks) { - if (task.getIngestJob().getJobId() == ingestJobId) { - return true; - } - } - - for (FileIngestTask task : directoryTasks) { - if (task.getIngestJob().getJobId() == ingestJobId) { - return true; - } - } - - for (FileIngestTask task : fileTasks) { - if (task.getIngestJob().getJobId() == ingestJobId) { - return true; - } - } - - return false; - } - - private void updateQueues() { + private void updateQueues() throws InterruptedException { // we loop because we could have a directory that has all files // that do not get enqueued while (true) { @@ -152,23 +120,15 @@ final class FileIngestTaskScheduler { if (rootDirectoryTasks.isEmpty()) { return; } - FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst(); - directoryTasks.add(rootTask); + addTaskToDirectoryQueue(rootDirectoryTasks.pollFirst(), false); } //pop and push AbstractFile directory children if any //add the popped and its leaf children onto cur file list FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); - final AbstractFile parentFile = parentTask.file; + final AbstractFile parentFile = parentTask.getFile(); // add itself to the file list if (shouldEnqueueTask(parentTask)) { - // RJCTODO - try { - parentTask.getIngestJob().notifyTaskPending(); - fileTasks.put(parentTask); - } catch (InterruptedException ex) { - // RJCTODO: Maybe make a convenience method - // RJCTODO: Need undo - } + addTaskToFileQueue(parentTask); } // add its children to the file and directory lists try { @@ -176,18 +136,11 @@ final class FileIngestTaskScheduler { for (Content c : children) { if (c instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) c; - FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getIngestJob()); + FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); if (childFile.hasChildren()) { - this.directoryTasks.add(childTask); + addTaskToDirectoryQueue(childTask, true); } else if (shouldEnqueueTask(childTask)) { - // RJCTODO - try { - childTask.getIngestJob().notifyTaskPending(); - fileTasks.put(childTask); - } catch (InterruptedException ex) { - // RJCTODO: Maybe make a convenience method - // RJCTODO: Need undo - } + addTaskToFileQueue(childTask); } } } @@ -197,10 +150,39 @@ final class FileIngestTaskScheduler { } } - synchronized void emptyQueues() { // RJCTODO: Perhaps clear all... - this.rootDirectoryTasks.clear(); - this.directoryTasks.clear(); - this.fileTasks.clear(); + private void addTaskToRootDirectoryQueue(FileIngestTask task) { + directoryTasks.add(task); + task.getIngestJob().notifyTaskAdded(); + } + + private void addTaskToDirectoryQueue(FileIngestTask task, boolean isNewTask) { + if (isNewTask) { + directoryTasks.add(task); + } + task.getIngestJob().notifyTaskAdded(); + } + + private void addTaskToFileQueue(FileIngestTask task) { + // The capacity of the file tasks queue is not bounded, so the call + // to put() should not block except for normal synchronized access. + // Still, notify the job that the task has been added first so that + // the take() of the task cannot occur before the notification. + task.getIngestJob().notifyTaskAdded(); + + // If the thread executing this code is ever interrupted, it is + // because the number of ingest threads has been decreased while + // ingest jobs are running. This thread will exit in an orderly fashion, + // but the task still needs to be enqueued rather than lost. + while (true) { + try { + fileTasks.put(task); + break; + } catch (InterruptedException ex) { + // Reset the interrupted status of the thread so the orderly + // exit can occur in the intended place. + Thread.currentThread().interrupt(); + } + } } /** @@ -211,7 +193,7 @@ final class FileIngestTaskScheduler { * @return true if should be enqueued, false otherwise */ private static boolean shouldEnqueueTask(final FileIngestTask processTask) { - final AbstractFile aFile = processTask.file; + final AbstractFile aFile = processTask.getFile(); //if it's unalloc file, skip if so scheduled if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { return false; @@ -320,10 +302,10 @@ final class FileIngestTaskScheduler { @Override public int compare(FileIngestTask q1, FileIngestTask q2) { - AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file); - AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file); + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile()); + AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile()); if (p1 == p2) { - return (int) (q2.file.getId() - q1.file.getId()); + return (int) (q2.getFile().getId() - q1.getFile().getId()); } else { return p2.ordinal() - p1.ordinal(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 8982be2061..a07c135e89 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -24,10 +24,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandleFactory; import org.openide.util.Cancellable; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; @@ -37,33 +39,48 @@ import org.sleuthkit.datamodel.Content; */ final class IngestJob { + private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final AtomicLong nextIngestJobId = new AtomicLong(0L); - private static final ConcurrentHashMap ingestJobs = new ConcurrentHashMap<>(); // Maps job ids to jobs. - private final long jobId; - private final Content dataSource; + private static final ConcurrentHashMap ingestJobsById = new ConcurrentHashMap<>(); + private final long id; + private final Content rootDataSource; private final List ingestModuleTemplates; private final boolean processUnallocatedSpace; private final LinkedBlockingQueue dataSourceIngestPipelines = new LinkedBlockingQueue<>(); private final LinkedBlockingQueue fileIngestPipelines = new LinkedBlockingQueue<>(); private final AtomicInteger tasksInProgress = new AtomicInteger(0); private final AtomicLong processedFiles = new AtomicLong(0L); + private final AtomicLong filesToIngestEstimate = new AtomicLong(0L); private ProgressHandle dataSourceTasksProgress; private ProgressHandle fileTasksProgress; - private long filesToIngestEstimate = 0; private volatile boolean cancelled; - static List startIngestJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { // RJCTODO: return errors + /** + * Creates an ingest job for a data source and starts up the ingest + * pipelines for the job. + * + * @param rootDataSource The data source to ingest. + * @param ingestModuleTemplates The ingest module templates to use to create + * the ingest pipelines for the job. + * @param processUnallocatedSpace Whether or not the job should include + * processing of unallocated space. + * @return A collection of ingest module startUp up errors, empty on + * success. + * @throws InterruptedException + */ + static List startJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { long jobId = nextIngestJobId.incrementAndGet(); IngestJob ingestJob = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); - List errors = ingestJob.start(); + List errors = ingestJob.startUp(); if (errors.isEmpty()) { - ingestJobs.put(jobId, ingestJob); + ingestJobsById.put(jobId, ingestJob); + IngestManager.getInstance().fireIngestJobStarted(jobId); } return errors; } static boolean jobsAreRunning() { - for (IngestJob job : ingestJobs.values()) { + for (IngestJob job : ingestJobsById.values()) { if (!job.isCancelled()) { return true; } @@ -71,79 +88,72 @@ final class IngestJob { return false; } - static void addFileToIngestJob(long ingestJobId, AbstractFile file) { // RJCTODO: Move back to IngestManager - IngestJob job = ingestJobs.get(ingestJobId); + static void addFileToJob(long ingestJobId, AbstractFile file) { // RJCTODO: Just one at a time? + IngestJob job = ingestJobsById.get(ingestJobId); if (job != null) { - FileIngestTaskScheduler.getInstance().addTask(job, file); +// long adjustedFilesCount = job.filesToIngestEstimate.incrementAndGet(); // RJCTODO: Not the best name now? +// job.fileTasksProgress.switchToIndeterminate(); // RJCTODO: Comment this stuff +// job.fileTasksProgress.switchToDeterminate((int) adjustedFilesCount); +// job.fileTasksProgress.progress(job.processedFiles.intValue()); + FileIngestTaskScheduler.getInstance().addTask(new FileIngestTask(job, file)); } } - static void cancelAllIngestJobs() { - for (IngestJob job : ingestJobs.values()) { + static void cancelAllJobs() { + for (IngestJob job : ingestJobsById.values()) { job.cancel(); } } private IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { - this.jobId = id; - this.dataSource = dataSource; + this.id = id; + this.rootDataSource = dataSource; this.ingestModuleTemplates = ingestModuleTemplates; this.processUnallocatedSpace = processUnallocatedSpace; this.cancelled = false; } - long getJobId() { - return jobId; - } - - Content getDataSource() { - return dataSource; + long getId() { + return id; } boolean shouldProcessUnallocatedSpace() { return processUnallocatedSpace; } - List start() { + List startUp() throws InterruptedException { List errors = startUpIngestPipelines(); if (errors.isEmpty()) { - DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, dataSource)); - FileIngestTaskScheduler.getInstance().addTasks(this, dataSource); startDataSourceIngestProgressBar(); startFileIngestProgressBar(); + FileIngestTaskScheduler.getInstance().addTasks(this, rootDataSource); // RJCTODO: Think about this ordering "solution" for small images + DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, rootDataSource)); } return errors; } - private List startUpIngestPipelines() { + private List startUpIngestPipelines() throws InterruptedException { + IngestJobContext context = new IngestJobContext(this); List errors = new ArrayList<>(); int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads(); for (int i = 0; i < maxNumberOfPipelines; ++i) { - DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); + DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates); errors.addAll(pipeline.startUp()); - try { - dataSourceIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: log unexpected block and interrupt, or throw - } - if (errors.isEmpty()) { - // No need to accumulate presumably redundant erros. + dataSourceIngestPipelines.put(pipeline); + if (!errors.isEmpty()) { + // No need to accumulate presumably redundant errors. break; } } maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads(); for (int i = 0; i < maxNumberOfPipelines; ++i) { - FileIngestPipeline pipeline = new FileIngestPipeline(this, ingestModuleTemplates); + FileIngestPipeline pipeline = new FileIngestPipeline(context, ingestModuleTemplates); errors.addAll(pipeline.startUp()); - try { - fileIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: log unexpected block and interrupt, or throw - } - if (errors.isEmpty()) { - // No need to accumulate presumably redundant erros. + fileIngestPipelines.put(pipeline); + if (!errors.isEmpty()) { + // No need to accumulate presumably redundant errors. break; } } @@ -154,7 +164,7 @@ final class IngestJob { private void startDataSourceIngestProgressBar() { final String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", - dataSource.getName()); + rootDataSource.getName()); dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { @@ -169,13 +179,13 @@ final class IngestJob { } }); dataSourceTasksProgress.start(); - dataSourceTasksProgress.switchToIndeterminate(); // RJCTODO: check out the logic in the pipleine class + dataSourceTasksProgress.switchToIndeterminate(); } private void startFileIngestProgressBar() { final String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", - dataSource.getName()); + rootDataSource.getName()); fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { @@ -188,77 +198,78 @@ final class IngestJob { return true; } }); - filesToIngestEstimate = dataSource.accept(new GetFilesCountVisitor()); + long initialFilesCount = rootDataSource.accept(new GetFilesCountVisitor()); + filesToIngestEstimate.getAndAdd(initialFilesCount); fileTasksProgress.start(); - fileTasksProgress.switchToDeterminate((int) filesToIngestEstimate); + fileTasksProgress.switchToDeterminate((int) initialFilesCount); // RJCTODO: This cast is troublesome, can use intValue } /** - * Called by the ingest task schedulers when an ingest task for this ingest - * job is added to the scheduler's task queue. + * Called by the ingest task schedulers when an ingest task is added to this + * ingest job. */ - void notifyTaskScheduled() { - // Increment the task counter when a task is scheduled so that there is - // a persistent record of the task's existence even after it is removed - // from the scheduler by an ingest thread. The task counter is used by - // the job to determine when it is done. + void notifyTaskAdded() { tasksInProgress.incrementAndGet(); } - /** - * Called by the ingest schedulers as an "undo" operation for - * notifyTaskScheduled(). - */ - void notifyTaskCompleted() { - // Decrement the task counter when a task is discarded by a scheduler. - // The task counter is used by the job to determine when it is done. - tasksInProgress.decrementAndGet(); + void process(Content dataSource) throws InterruptedException { + // If the job is not cancelled, complete the task, otherwise just flush + // it. In either case, the task counter needs to be decremented and the + // shut down check needs to occur. + if (!isCancelled()) { + List errors = new ArrayList<>(); + DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take(); + errors.addAll(pipeline.process(dataSource, dataSourceTasksProgress)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + dataSourceIngestPipelines.put(pipeline); + } + shutDownIfAllTasksCompleted(); } - void process() throws InterruptedException { + void process(AbstractFile file) throws InterruptedException { + // If the job is not cancelled, complete the task, otherwise just flush + // it. In either case, the task counter needs to be decremented and the + // shut down check needs to occur. if (!isCancelled()) { - try { - DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take(); - pipeline.process(); // RJCTODO: Pass data source through? - dataSourceIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: + List errors = new ArrayList<>(); + FileIngestPipeline pipeline = fileIngestPipelines.take(); +// fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); RJCTODO + errors.addAll(pipeline.process(file)); + fileIngestPipelines.put(pipeline); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); } } - ifCompletedShutDown(); + shutDownIfAllTasksCompleted(); } - void process(AbstractFile file) { - if (!isCancelled()) { - try { - FileIngestPipeline pipeline = fileIngestPipelines.take(); - fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); - pipeline.process(file); - fileIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: Log block and interrupt - } - } - ifCompletedShutDown(); - } - - void ifCompletedShutDown() { + private void shutDownIfAllTasksCompleted() { if (tasksInProgress.decrementAndGet() == 0) { + List errors = new ArrayList<>(); while (!dataSourceIngestPipelines.isEmpty()) { DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll(); - pipeline.shutDown(cancelled); + errors.addAll(pipeline.shutDown()); } while (!fileIngestPipelines.isEmpty()) { FileIngestPipeline pipeline = fileIngestPipelines.poll(); - pipeline.shutDown(cancelled); + errors.addAll(pipeline.shutDown()); } - ingestJobs.remove(jobId); - IngestManager.getInstance().fireIngestJobCompleted(jobId); + fileTasksProgress.finish(); + dataSourceTasksProgress.finish(); + ingestJobsById.remove(id); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + IngestManager.getInstance().fireIngestJobCompleted(id); } } - ProgressHandle getDataSourceTaskProgressBar() { - return dataSourceTasksProgress; // RJCTODO: Should just pass the progress handle or the object to the pipeline + private void logIngestModuleErrors(List errors) { + for (IngestModuleError error : errors) { + logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); + } } boolean isCancelled() { @@ -267,7 +278,7 @@ final class IngestJob { void cancel() { cancelled = true; - fileTasksProgress.finish(); - IngestManager.getInstance().fireIngestJobCancelled(jobId); + fileTasksProgress.finish(); // RJCTODO: What about the other progress bar? + IngestManager.getInstance().fireIngestJobCancelled(id); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 6a9b4a9454..b8d4493de4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -39,7 +39,7 @@ public final class IngestJobContext { * @return The ingest job identifier. */ public long getJobId() { - return this.ingestJob.getJobId(); + return this.ingestJob.getId(); } /** @@ -60,7 +60,7 @@ public final class IngestJobContext { */ public void addFiles(List files) { for (AbstractFile file : files) { - IngestJob.addFileToIngestJob(ingestJob.getJobId(), file); + IngestJob.addFileToJob(ingestJob.getId(), file); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index a5b0da296c..36cc73fefb 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -21,7 +21,9 @@ package org.sleuthkit.autopsy.ingest; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.util.List; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -51,14 +53,15 @@ public class IngestManager { private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final IngestManager instance = new IngestManager(); - private final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); + private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); + private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); - private final ConcurrentHashMap> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); - private final ConcurrentHashMap> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); - private final ExecutorService fireIngestJobEventsThreadPool = Executors.newSingleThreadExecutor(); + private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); + private final ConcurrentHashMap> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. + private final ConcurrentHashMap> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final AtomicLong nextThreadId = new AtomicLong(0L); private volatile IngestMessageTopComponent ingestMessageBox; @@ -154,7 +157,7 @@ public class IngestManager { */ private void startDataSourceIngestThread() { long threadId = nextThreadId.incrementAndGet(); - Future handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread(threadId)); + Future handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread()); dataSourceIngestThreads.put(threadId, handle); } @@ -164,7 +167,7 @@ public class IngestManager { */ private void startFileIngestThread() { long threadId = nextThreadId.incrementAndGet(); - Future handle = fileIngestThreadPool.submit(new FileIngestThread(threadId)); + Future handle = fileIngestThreadPool.submit(new FileIngestThread()); fileIngestThreads.put(threadId, handle); } @@ -200,35 +203,48 @@ public class IngestManager { } public void cancelAllIngestJobs() { - cancelStartIngestJobsTasks(); - IngestJob.cancelAllIngestJobs(); - } - - private void cancelStartIngestJobsTasks() { - for (Future future : startIngestJobThreads.values()) { - future.cancel(true); + // Stop creating new ingest jobs. + for (Future handle : startIngestJobThreads.values()) { + handle.cancel(true); + try { + // Blocks until the job starting thread responds. The thread + // removes itself from this collection, which does not disrupt + // this loop since the collection is a ConcurrentHashMap. + handle.get(); + } catch (InterruptedException | ExecutionException ex) { + // This should never happen, something is awry, but everything + // should be o.k. anyway. + logger.log(Level.SEVERE, "Unexpected thread interrupt", ex); + } } - startIngestJobThreads.clear(); + startIngestJobThreads.clear(); // Make sure. + + // Cancel all the jobs already created. This will make the the ingest + // threads flush out any lingering ingest tasks without processing them. + IngestJob.cancelAllJobs(); } /** * Ingest events. */ - public enum IngestEvent { // RJCTODO: Update comments if time permits + public enum IngestEvent { /** * Property change event fired when an ingest job is started. The old - * and new values of the PropertyChangeEvent object are set to null. + * value of the PropertyChangeEvent object is set to the ingest job id, + * and the new value is set to null. */ INGEST_JOB_STARTED, /** * Property change event fired when an ingest job is completed. The old - * and new values of the PropertyChangeEvent object are set to null. + * value of the PropertyChangeEvent object is set to the ingest job id, + * and the new value is set to null. */ INGEST_JOB_COMPLETED, /** * Property change event fired when an ingest job is canceled. The old - * and new values of the PropertyChangeEvent object are set to null. + * value of the PropertyChangeEvent object is set to the ingest job id, + * and the new value is set to null. */ INGEST_JOB_CANCELLED, /** @@ -253,21 +269,59 @@ public class IngestManager { }; /** - * Add an ingest event property change listener. + * Add an ingest job event property change listener. * * @param listener The PropertyChangeListener to register. */ - public void addPropertyChangeListener(final PropertyChangeListener listener) { - pcs.addPropertyChangeListener(listener); + public void addIngestJobEventListener(final PropertyChangeListener listener) { + ingestJobEventPublisher.addPropertyChangeListener(listener); } /** - * Remove an ingest event property change listener. + * Remove an ingest job event property change listener. * * @param listener The PropertyChangeListener to unregister. */ - public void removePropertyChangeListener(final PropertyChangeListener listener) { - pcs.removePropertyChangeListener(listener); + public void removeIngestJobEventListener(final PropertyChangeListener listener) { + ingestJobEventPublisher.removePropertyChangeListener(listener); + } + + /** + * Add an ingest module event property change listener. + * + * @param listener The PropertyChangeListener to register. + */ + public void addIngestModuleEventListener(final PropertyChangeListener listener) { + ingestModuleEventPublisher.addPropertyChangeListener(listener); + } + + /** + * Remove an ingest module event property change listener. + * + * @param listener The PropertyChangeListener to unregister. + */ + public void removeIngestModuleEventListener(final PropertyChangeListener listener) { + ingestModuleEventPublisher.removePropertyChangeListener(listener); + } + + /** + * Add an ingest module event property change listener. + * + * @deprecated + * @param listener The PropertyChangeListener to register. + */ + public static void addPropertyChangeListener(final PropertyChangeListener listener) { + instance.ingestModuleEventPublisher.addPropertyChangeListener(listener); + } + + /** + * Remove an ingest module event property change listener. + * + * @deprecated + * @param listener The PropertyChangeListener to unregister. + */ + public static void removePropertyChangeListener(final PropertyChangeListener listener) { + instance.ingestModuleEventPublisher.removePropertyChangeListener(listener); } /** @@ -276,7 +330,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobStarted(long ingestJobId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_STARTED, ingestJobId, null)); } /** @@ -285,7 +339,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCompleted(long ingestJobId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null)); } /** @@ -294,7 +348,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCancelled(long ingestJobId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null)); } /** @@ -303,7 +357,7 @@ public class IngestManager { * @param fileId The object id of file. */ void fireFileIngestDone(long fileId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.FILE_DONE, fileId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.FILE_DONE, fileId, null)); } /** @@ -312,7 +366,7 @@ public class IngestManager { * @param moduleDataEvent A ModuleDataEvent with the details of the posting. */ void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.DATA, moduleDataEvent, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.DATA, moduleDataEvent, null)); } /** @@ -323,7 +377,7 @@ public class IngestManager { * content. */ void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.CONTENT_CHANGED, moduleContentEvent, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.CONTENT_CHANGED, moduleContentEvent, null)); } /** @@ -341,7 +395,7 @@ public class IngestManager { * Get the free disk space of the drive where to which ingest data is being * written, as reported by the ingest monitor. * - * @return Free disk space, -1 if unknown // RJCTODO: What units? + * @return Free disk space, -1 if unknown */ long getFreeDiskSpace() { if (ingestMonitor != null) { @@ -352,10 +406,10 @@ public class IngestManager { } /** - * A Runnable that creates ingest jobs and submits the initial data source + * A Callable that creates ingest jobs and submits the initial data source * and file ingest tasks to the task schedulers. */ - private class StartIngestJobsThread implements Runnable { + private class StartIngestJobsThread implements Callable { private final long threadId; private final List dataSources; @@ -371,7 +425,7 @@ public class IngestManager { } @Override - public void run() { + public Void call() { try { final String displayName = NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.displayName"); @@ -387,26 +441,25 @@ public class IngestManager { return true; } }); - progress.start(dataSources.size() * 2); + progress.start(dataSources.size()); if (!ingestMonitor.isRunning()) { ingestMonitor.start(); } - int workUnitsCompleted = 0; + int dataSourceProcessed = 0; for (Content dataSource : dataSources) { if (Thread.currentThread().isInterrupted()) { break; } - // Create an ingest job. - List errors = IngestJob.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace); + // Start an ingest job for the data source. + List errors = IngestJob.startJob(dataSource, moduleTemplates, processUnallocatedSpace); if (!errors.isEmpty()) { - // Report the error to the user. + // Report the errors to the user. They have already been logged. StringBuilder moduleStartUpErrors = new StringBuilder(); for (IngestModuleError error : errors) { String moduleName = error.getModuleDisplayName(); - logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); //NON-NLS moduleStartUpErrors.append(moduleName); moduleStartUpErrors.append(": "); moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); @@ -427,40 +480,16 @@ public class IngestManager { NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); } - - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED)); - - // Queue a data source ingest task for the ingest job. - final String inputName = dataSource.getName(); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg1", - inputName), workUnitsCompleted); - DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(ingestJob, ingestJob.getDataSource())); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg2", - inputName), ++workUnitsCompleted); - - // Queue the file ingest tasks for the ingest job. - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg3", - inputName), workUnitsCompleted); - FileIngestTaskScheduler.getInstance().addTasks(ingestJob, ingestJob.getDataSource()); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg4", - inputName), ++workUnitsCompleted); + progress.progress(++dataSourceProcessed); if (!Thread.currentThread().isInterrupted()) { break; } } - } catch (Exception ex) { - String message = String.format("StartIngestJobsTask (id=%d) caught exception", threadId); //NON-NLS - logger.log(Level.SEVERE, message, ex); - MessageNotifyUtil.Message.error( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg")); } finally { progress.finish(); startIngestJobThreads.remove(threadId); + return null; } } } @@ -512,16 +541,18 @@ public class IngestManager { } /** - * A Runnable that fire ingest events to ingest manager property change + * A Runnable that fires ingest events to ingest manager property change * listeners. */ - private class FireIngestEventThread implements Runnable { + private static class FireIngestEventThread implements Runnable { + private final PropertyChangeSupport publisher; private final IngestEvent event; private final Object oldValue; private final Object newValue; - FireIngestEventThread(IngestEvent event, Object oldValue, Object newValue) { + FireIngestEventThread(PropertyChangeSupport publisher, IngestEvent event, Object oldValue, Object newValue) { + this.publisher = publisher; this.event = event; this.oldValue = oldValue; this.newValue = newValue; @@ -530,10 +561,10 @@ public class IngestManager { @Override public void run() { try { - pcs.firePropertyChange(event.toString(), oldValue, newValue); + publisher.firePropertyChange(event.toString(), oldValue, newValue); } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), // RJCTODO: Oddly named strings + MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), MessageNotifyUtil.MessageType.ERROR); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java index c573753281..5bf4f182a7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java @@ -95,7 +95,7 @@ public final class IngestServices { * artifact data */ public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { - IngestManager.fireIngestModuleDataEvent(moduleDataEvent); + IngestManager.getInstance().fireIngestModuleDataEvent(moduleDataEvent); } /** @@ -107,7 +107,7 @@ public final class IngestServices { * changed */ public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { - IngestManager.fireIngestModuleContentEvent(moduleContentEvent); + IngestManager.getInstance().fireIngestModuleContentEvent(moduleContentEvent); } /** diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java index 7e901fbc71..c414731036 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java @@ -69,7 +69,7 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan // Listen to the ingest modules to refresh the enabled/disabled state of // the components in sync with file ingest. - IngestManager.addPropertyChangeListener(new PropertyChangeListener() { + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { if (isIngestJobEvent(evt)) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java index 90e7f197d1..4ba2d39eb6 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java @@ -126,7 +126,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec setButtonStates(); - IngestManager.addPropertyChangeListener(new PropertyChangeListener() { + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java index 898b21bfc5..0e8c099aac 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java @@ -118,7 +118,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { ingestRunning = IngestManager.getInstance().isIngestRunning(); updateComponents(); - IngestManager.addPropertyChangeListener(new PropertyChangeListener() { + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); diff --git a/c:casesSamll2Againautopsy.db b/c:casesSamll2Againautopsy.db new file mode 100755 index 0000000000..e69de29bb2 From 6f0a440ca05a74d1c5f43a3fbce4499688288acf Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 1 May 2014 13:49:56 -0400 Subject: [PATCH 3/5] Fixed some bugs in management of IngestJob task counter --- .../autopsy/ingest/FileIngestTask.java | 7 ++-- .../ingest/FileIngestTaskScheduler.java | 38 ++++++++----------- .../autopsy/ingest/GetFilesCountVisitor.java | 1 + .../sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../sleuthkit/autopsy/ingest/IngestTask.java | 23 +++++++++++ 5 files changed, 44 insertions(+), 27 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 54ed9a8501..75e9165ded 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -21,7 +21,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.Objects; import org.sleuthkit.datamodel.AbstractFile; -final class FileIngestTask { +final class FileIngestTask implements IngestTask { private final IngestJob ingestJob; private final AbstractFile file; @@ -31,7 +31,7 @@ final class FileIngestTask { this.file = file; } - public IngestJob getIngestJob() { + public IngestJob getIngestJob() { // RJCTODO: Maybe add to interface return ingestJob; } @@ -39,7 +39,8 @@ final class FileIngestTask { return file; } - void execute() throws InterruptedException { + @Override + public void execute() throws InterruptedException { ingestJob.process(file); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java index fcbceab6ec..e0da1c0829 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java @@ -86,7 +86,8 @@ final class FileIngestTaskScheduler { for (AbstractFile firstLevelFile : firstLevelFiles) { FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); if (shouldEnqueueTask(fileTask)) { - addTaskToRootDirectoryQueue(fileTask); + rootDirectoryTasks.add(fileTask); + fileTask.getIngestJob().notifyTaskAdded(); } } @@ -96,7 +97,7 @@ final class FileIngestTaskScheduler { synchronized void addTask(FileIngestTask task) { if (shouldEnqueueTask(task)) { - addTaskToFileQueue(task); + addTaskToFileQueue(task, true); } } @@ -120,7 +121,8 @@ final class FileIngestTaskScheduler { if (rootDirectoryTasks.isEmpty()) { return; } - addTaskToDirectoryQueue(rootDirectoryTasks.pollFirst(), false); + FileIngestTask rootTask = rootDirectoryTasks.pollFirst(); + directoryTasks.add(rootTask); } //pop and push AbstractFile directory children if any //add the popped and its leaf children onto cur file list @@ -128,7 +130,7 @@ final class FileIngestTaskScheduler { final AbstractFile parentFile = parentTask.getFile(); // add itself to the file list if (shouldEnqueueTask(parentTask)) { - addTaskToFileQueue(parentTask); + addTaskToFileQueue(parentTask, false); } // add its children to the file and directory lists try { @@ -138,9 +140,10 @@ final class FileIngestTaskScheduler { AbstractFile childFile = (AbstractFile) c; FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); if (childFile.hasChildren()) { - addTaskToDirectoryQueue(childTask, true); + directoryTasks.add(childTask); + childTask.getIngestJob().notifyTaskAdded(); } else if (shouldEnqueueTask(childTask)) { - addTaskToFileQueue(childTask); + addTaskToFileQueue(childTask, true); } } } @@ -150,25 +153,14 @@ final class FileIngestTaskScheduler { } } - private void addTaskToRootDirectoryQueue(FileIngestTask task) { - directoryTasks.add(task); - task.getIngestJob().notifyTaskAdded(); - } - - private void addTaskToDirectoryQueue(FileIngestTask task, boolean isNewTask) { + private void addTaskToFileQueue(FileIngestTask task, boolean isNewTask) { if (isNewTask) { - directoryTasks.add(task); + // The capacity of the file tasks queue is not bounded, so the call + // to put() should not block except for normal synchronized access. + // Still, notify the job that the task has been added first so that + // the take() of the task cannot occur before the notification. + task.getIngestJob().notifyTaskAdded(); } - task.getIngestJob().notifyTaskAdded(); - } - - private void addTaskToFileQueue(FileIngestTask task) { - // The capacity of the file tasks queue is not bounded, so the call - // to put() should not block except for normal synchronized access. - // Still, notify the job that the task has been added first so that - // the take() of the task cannot occur before the notification. - task.getIngestJob().notifyTaskAdded(); - // If the thread executing this code is ever interrupted, it is // because the number of ingest threads has been decreased while // ingest jobs are running. This thread will exit in an orderly fashion, diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java index 346da5d418..f8c280628d 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java @@ -51,6 +51,7 @@ final class GetFilesCountVisitor extends ContentVisitor.Default { queryB.append(") )"); queryB.append(" AND ( (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS + queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_VIRT.getValue()); //NON-NLS queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS queryB.append(") )"); //queryB.append( "AND (type = "); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index a07c135e89..424aa638a2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -235,7 +235,7 @@ final class IngestJob { if (!isCancelled()) { List errors = new ArrayList<>(); FileIngestPipeline pipeline = fileIngestPipelines.take(); -// fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); RJCTODO + fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); errors.addAll(pipeline.process(file)); fileIngestPipelines.put(pipeline); if (!errors.isEmpty()) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java new file mode 100755 index 0000000000..1133be1210 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -0,0 +1,23 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +interface IngestTask { + void execute() throws InterruptedException; +} From 07e9956ee0f4f2692c12783160bd7c6f3f2d5547 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 1 May 2014 17:46:35 -0400 Subject: [PATCH 4/5] Refactor ingest framework --- .../AddImageWizardIngestConfigPanel.java | 6 +- .../autopsy/ingest/DataSourceIngestTask.java | 5 +- .../ingest/DataSourceIngestTaskScheduler.java | 73 +- .../ingest/FileIngestTaskScheduler.java | 701 +++++++++--------- .../autopsy/ingest/GetFilesCountVisitor.java | 2 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 55 +- ...uncher.java => IngestJobConfigurator.java} | 6 +- .../autopsy/ingest/IngestJobContext.java | 2 +- .../autopsy/ingest/IngestJobScheduler.java | 462 ++++++++++++ .../autopsy/ingest/IngestManager.java | 54 +- .../autopsy/ingest/IngestTaskQueue.java | 23 + .../ingest/RunIngestModulesDialog.java | 4 +- .../AddContentToHashDbAction.java | 2 +- 13 files changed, 902 insertions(+), 493 deletions(-) rename Core/src/org/sleuthkit/autopsy/ingest/{IngestJobLauncher.java => IngestJobConfigurator.java} (99%) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/IngestTaskQueue.java diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java index a36c061bac..c5cc14d262 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageWizardIngestConfigPanel.java @@ -19,7 +19,7 @@ package org.sleuthkit.autopsy.casemodule; -import org.sleuthkit.autopsy.ingest.IngestJobLauncher; +import org.sleuthkit.autopsy.ingest.IngestJobConfigurator; import org.openide.util.NbBundle; import java.awt.Color; import java.awt.Component; @@ -46,7 +46,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel { private static final Logger logger = Logger.getLogger(AddImageWizardIngestConfigPanel.class.getName()); - private IngestJobLauncher ingestConfig; + private IngestJobConfigurator ingestConfig; /** * The visual component that displays this panel. If you need to access the * component from this class, just use getComponent(). @@ -73,7 +73,7 @@ class AddImageWizardIngestConfigPanel implements WizardDescriptor.Panel messages = ingestConfig.getIngestJobConfigWarnings(); if (messages.isEmpty() == false) { StringBuilder warning = new StringBuilder(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index 2ad3c644e3..c62b6796a1 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.Content; -final class DataSourceIngestTask { +final class DataSourceIngestTask implements IngestTask { private final IngestJob ingestJob; private final Content dataSource; @@ -38,7 +38,8 @@ final class DataSourceIngestTask { return dataSource; } - void execute() throws InterruptedException { + @Override + public void execute() throws InterruptedException { ingestJob.process(dataSource); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java index ca22a6267b..ba89a669a4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java @@ -20,42 +20,37 @@ package org.sleuthkit.autopsy.ingest; import java.util.concurrent.LinkedBlockingQueue; -final class DataSourceIngestTaskScheduler { - - private static DataSourceIngestTaskScheduler instance = new DataSourceIngestTaskScheduler(); - private final LinkedBlockingQueue tasks = new LinkedBlockingQueue<>(); - - static DataSourceIngestTaskScheduler getInstance() { - return instance; - } - - private DataSourceIngestTaskScheduler() { - } - - synchronized void addTask(DataSourceIngestTask task) { - // The capacity of the tasks queue is not bounded, so the call - // to put() should not block except for normal synchronized access. - // Still, notify the job that the task has been added first so that - // the take() of the task cannot occur before the notification. - task.getIngestJob().notifyTaskAdded(); - - // If the thread executing this code is ever interrupted, it is - // because the number of ingest threads has been decreased while - // ingest jobs are running. This thread will exit in an orderly fashion, - // but the task still needs to be enqueued rather than lost. - while (true) { - try { - tasks.put(task); - break; - } catch (InterruptedException ex) { - // Reset the interrupted status of the thread so the orderly - // exit can occur in the intended place. - Thread.currentThread().interrupt(); - } - } - } - - DataSourceIngestTask getNextTask() throws InterruptedException { - return tasks.take(); - } -} +//final class DataSourceIngestTaskScheduler implements IngestTaskQueue { +// +// private static DataSourceIngestTaskScheduler instance = new DataSourceIngestTaskScheduler(); +// private final LinkedBlockingQueue tasks = new LinkedBlockingQueue<>(); +// +// static DataSourceIngestTaskScheduler getInstance() { +// return instance; +// } +// +// private DataSourceIngestTaskScheduler() { +// } +// +// void addTask(DataSourceIngestTask task) { +// // If the thread executing this code is interrupted, it is because the +// // number of ingest threads has been decreased while ingest jobs are +// // running. This thread will exit in an orderly fashion, but the task +// // still needs to be enqueued rather than lost. +// while (true) { +// try { +// tasks.put(task); +// break; +// } catch (InterruptedException ex) { +// // Reset the interrupted status of the thread so the orderly +// // exit can occur in the intended place. +// Thread.currentThread().interrupt(); +// } +// } +// } +// +// @Override +// public DataSourceIngestTask getNextTask() throws InterruptedException { +// return tasks.take(); +// } +//} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java index e0da1c0829..c0a4b34295 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java @@ -40,353 +40,354 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.VirtualDirectory; -final class FileIngestTaskScheduler { - - private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); - private static FileIngestTaskScheduler instance = new FileIngestTaskScheduler(); - private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); - private final List directoryTasks = new ArrayList<>(); - private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); - private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); - - static FileIngestTaskScheduler getInstance() { - return instance; - } - - private FileIngestTaskScheduler() { - } - - synchronized void addTasks(IngestJob job, Content dataSource) throws InterruptedException { - Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); - List firstLevelFiles = new ArrayList<>(); - if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { - // The data source is file. - firstLevelFiles.add((AbstractFile) dataSource); - } else { - for (AbstractFile root : rootObjects) { - List children; - try { - children = root.getChildren(); - if (children.isEmpty()) { - //add the root itself, could be unalloc file, child of volume or image - firstLevelFiles.add(root); - } else { - //root for fs root dir, schedule children dirs/files - for (Content child : children) { - if (child instanceof AbstractFile) { - firstLevelFiles.add((AbstractFile) child); - } - } - } - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS - } - } - } - for (AbstractFile firstLevelFile : firstLevelFiles) { - FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); - if (shouldEnqueueTask(fileTask)) { - rootDirectoryTasks.add(fileTask); - fileTask.getIngestJob().notifyTaskAdded(); - } - } - - // Reshuffle/update the dir and file level queues if needed - updateQueues(); - } - - synchronized void addTask(FileIngestTask task) { - if (shouldEnqueueTask(task)) { - addTaskToFileQueue(task, true); - } - } - - FileIngestTask getNextTask() throws InterruptedException { - FileIngestTask task = fileTasks.take(); - updateQueues(); - return task; - } - - private void updateQueues() throws InterruptedException { - // we loop because we could have a directory that has all files - // that do not get enqueued - while (true) { - // There are files in the queue, we're done - if (fileTasks.isEmpty() == false) { - return; - } - // fill in the directory queue if it is empty. - if (this.directoryTasks.isEmpty()) { - // bail out if root is also empty -- we are done - if (rootDirectoryTasks.isEmpty()) { - return; - } - FileIngestTask rootTask = rootDirectoryTasks.pollFirst(); - directoryTasks.add(rootTask); - } - //pop and push AbstractFile directory children if any - //add the popped and its leaf children onto cur file list - FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); - final AbstractFile parentFile = parentTask.getFile(); - // add itself to the file list - if (shouldEnqueueTask(parentTask)) { - addTaskToFileQueue(parentTask, false); - } - // add its children to the file and directory lists - try { - List children = parentFile.getChildren(); - for (Content c : children) { - if (c instanceof AbstractFile) { - AbstractFile childFile = (AbstractFile) c; - FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); - if (childFile.hasChildren()) { - directoryTasks.add(childTask); - childTask.getIngestJob().notifyTaskAdded(); - } else if (shouldEnqueueTask(childTask)) { - addTaskToFileQueue(childTask, true); - } - } - } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex); - } - } - } - - private void addTaskToFileQueue(FileIngestTask task, boolean isNewTask) { - if (isNewTask) { - // The capacity of the file tasks queue is not bounded, so the call - // to put() should not block except for normal synchronized access. - // Still, notify the job that the task has been added first so that - // the take() of the task cannot occur before the notification. - task.getIngestJob().notifyTaskAdded(); - } - // If the thread executing this code is ever interrupted, it is - // because the number of ingest threads has been decreased while - // ingest jobs are running. This thread will exit in an orderly fashion, - // but the task still needs to be enqueued rather than lost. - while (true) { - try { - fileTasks.put(task); - break; - } catch (InterruptedException ex) { - // Reset the interrupted status of the thread so the orderly - // exit can occur in the intended place. - Thread.currentThread().interrupt(); - } - } - } - - /** - * Check if the file is a special file that we should skip - * - * @param processTask a task whose file to check if should be queued of - * skipped - * @return true if should be enqueued, false otherwise - */ - private static boolean shouldEnqueueTask(final FileIngestTask processTask) { - final AbstractFile aFile = processTask.getFile(); - //if it's unalloc file, skip if so scheduled - if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { - return false; - } - String fileName = aFile.getName(); - if (fileName.equals(".") || fileName.equals("..")) { - return false; - } else if (aFile instanceof org.sleuthkit.datamodel.File) { - final org.sleuthkit.datamodel.File f = (File) aFile; - //skip files in root dir, starting with $, containing : (not default attributes) - //with meta address < 32, i.e. some special large NTFS and FAT files - FileSystem fs = null; - try { - fs = f.getFileSystem(); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS - } - TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP; - if (fs != null) { - fsType = fs.getFsType(); - } - if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { - //not fat or ntfs, accept all files - return true; - } - boolean isInRootDir = false; - try { - isInRootDir = f.getParentDirectory().isRoot(); - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS - } - if (isInRootDir && f.getMetaAddr() < 32) { - String name = f.getName(); - if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) { - return false; - } - } else { - return true; - } - } - return true; - } - - /** - * Visitor that gets a collection of top level objects to be scheduled, such - * as root directories (if there is FS) or LayoutFiles and virtual - * directories, also if there is no FS. - */ - static class GetRootDirectoryVisitor extends GetFilesContentVisitor { - - @Override - public Collection visit(VirtualDirectory ld) { - //case when we hit a layout directoryor local file container, not under a real FS - //or when root virt dir is scheduled - Collection ret = new ArrayList<>(); - ret.add(ld); - return ret; - } - - @Override - public Collection visit(LayoutFile lf) { - //case when we hit a layout file, not under a real FS - Collection ret = new ArrayList<>(); - ret.add(lf); - return ret; - } - - @Override - public Collection visit(Directory drctr) { - //we hit a real directory, a child of real FS - Collection ret = new ArrayList<>(); - ret.add(drctr); - return ret; - } - - @Override - public Collection visit(FileSystem fs) { - return getAllFromChildren(fs); - } - - @Override - public Collection visit(File file) { - //can have derived files - return getAllFromChildren(file); - } - - @Override - public Collection visit(DerivedFile derivedFile) { - //can have derived files - //TODO test this and overall scheduler with derived files - return getAllFromChildren(derivedFile); - } - - @Override - public Collection visit(LocalFile localFile) { - //can have local files - //TODO test this and overall scheduler with local files - return getAllFromChildren(localFile); - } - } - - /** - * Root directory sorter - */ - private static class RootDirectoryTaskComparator implements Comparator { - - @Override - public int compare(FileIngestTask q1, FileIngestTask q2) { - AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile()); - AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile()); - if (p1 == p2) { - return (int) (q2.getFile().getId() - q1.getFile().getId()); - } else { - return p2.ordinal() - p1.ordinal(); - } - } - - /** - * Priority determination for sorted AbstractFile, used by - * RootDirComparator - */ - private static class AbstractFilePriority { - - enum Priority { - - LAST, LOW, MEDIUM, HIGH - } - static final List LAST_PRI_PATHS = new ArrayList<>(); - static final List LOW_PRI_PATHS = new ArrayList<>(); - static final List MEDIUM_PRI_PATHS = new ArrayList<>(); - static final List HIGH_PRI_PATHS = new ArrayList<>(); - /* prioritize root directory folders based on the assumption that we are - * looking for user content. Other types of investigations may want different - * priorities. */ - - static /* prioritize root directory folders based on the assumption that we are - * looking for user content. Other types of investigations may want different - * priorities. */ { - // these files have no structure, so they go last - //unalloc files are handled as virtual files in getPriority() - //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); - //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE)); - LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE)); - LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE)); - // orphan files are often corrupt and windows does not typically have - // user content, so put them towards the bottom - LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE)); - LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE)); - // all other files go into the medium category too - MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE)); - // user content is top priority - HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE)); - HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE)); - HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE)); - HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE)); - } - - /** - * Get the scheduling priority for a given file. - * - * @param abstractFile - * @return - */ - static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) { - if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { - //quickly filter out unstructured content - //non-fs virtual files and dirs, such as representing unalloc space - return AbstractFilePriority.Priority.LAST; - } - //determine the fs files priority by name - final String path = abstractFile.getName(); - if (path == null) { - return AbstractFilePriority.Priority.MEDIUM; - } - for (Pattern p : HIGH_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.HIGH; - } - } - for (Pattern p : MEDIUM_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.MEDIUM; - } - } - for (Pattern p : LOW_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.LOW; - } - } - for (Pattern p : LAST_PRI_PATHS) { - Matcher m = p.matcher(path); - if (m.find()) { - return AbstractFilePriority.Priority.LAST; - } - } - //default is medium - return AbstractFilePriority.Priority.MEDIUM; - } - } - } -} +//final class FileIngestTaskScheduler implements IngestTaskQueue { +// +// private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); +// private static FileIngestTaskScheduler instance = new FileIngestTaskScheduler(); +// private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); +// private final List directoryTasks = new ArrayList<>(); +// private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); +// private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); +// +// static FileIngestTaskScheduler getInstance() { +// return instance; +// } +// +// private FileIngestTaskScheduler() { +// } +// +// synchronized void addTasks(IngestJob job, Content dataSource) throws InterruptedException { +// Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); +// List firstLevelFiles = new ArrayList<>(); +// if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { +// // The data source is file. +// firstLevelFiles.add((AbstractFile) dataSource); +// } else { +// for (AbstractFile root : rootObjects) { +// List children; +// try { +// children = root.getChildren(); +// if (children.isEmpty()) { +// //add the root itself, could be unalloc file, child of volume or image +// firstLevelFiles.add(root); +// } else { +// //root for fs root dir, schedule children dirs/files +// for (Content child : children) { +// if (child instanceof AbstractFile) { +// firstLevelFiles.add((AbstractFile) child); +// } +// } +// } +// } catch (TskCoreException ex) { +// logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS +// } +// } +// } +// for (AbstractFile firstLevelFile : firstLevelFiles) { +// FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); +// if (shouldEnqueueTask(fileTask)) { +// rootDirectoryTasks.add(fileTask); +// fileTask.getIngestJob().notifyTaskAdded(); +// } +// } +// +// // Reshuffle/update the dir and file level queues if needed +// updateQueues(); +// } +// +// synchronized void addTask(FileIngestTask task) { +// if (shouldEnqueueTask(task)) { +// addTaskToFileQueue(task, true); +// } +// } +// +// @Override +// public FileIngestTask getNextTask() throws InterruptedException { +// FileIngestTask task = fileTasks.take(); +// updateQueues(); +// return task; +// } +// +// private void updateQueues() throws InterruptedException { +// // we loop because we could have a directory that has all files +// // that do not get enqueued +// while (true) { +// // There are files in the queue, we're done +// if (fileTasks.isEmpty() == false) { +// return; +// } +// // fill in the directory queue if it is empty. +// if (this.directoryTasks.isEmpty()) { +// // bail out if root is also empty -- we are done +// if (rootDirectoryTasks.isEmpty()) { +// return; +// } +// FileIngestTask rootTask = rootDirectoryTasks.pollFirst(); +// directoryTasks.add(rootTask); +// } +// //pop and push AbstractFile directory children if any +// //add the popped and its leaf children onto cur file list +// FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); +// final AbstractFile parentFile = parentTask.getFile(); +// // add itself to the file list +// if (shouldEnqueueTask(parentTask)) { +// addTaskToFileQueue(parentTask, false); +// } +// // add its children to the file and directory lists +// try { +// List children = parentFile.getChildren(); +// for (Content c : children) { +// if (c instanceof AbstractFile) { +// AbstractFile childFile = (AbstractFile) c; +// FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); +// if (childFile.hasChildren()) { +// directoryTasks.add(childTask); +// childTask.getIngestJob().notifyTaskAdded(); +// } else if (shouldEnqueueTask(childTask)) { +// addTaskToFileQueue(childTask, true); +// } +// } +// } +// } catch (TskCoreException ex) { +// logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex); +// } +// } +// } +// +// private void addTaskToFileQueue(FileIngestTask task, boolean isNewTask) { +// if (isNewTask) { +// // The capacity of the file tasks queue is not bounded, so the call +// // to put() should not block except for normal synchronized access. +// // Still, notify the job that the task has been added first so that +// // the take() of the task cannot occur before the notification. +// task.getIngestJob().notifyTaskAdded(); +// } +// // If the thread executing this code is ever interrupted, it is +// // because the number of ingest threads has been decreased while +// // ingest jobs are running. This thread will exit in an orderly fashion, +// // but the task still needs to be enqueued rather than lost. +// while (true) { +// try { +// fileTasks.put(task); +// break; +// } catch (InterruptedException ex) { +// // Reset the interrupted status of the thread so the orderly +// // exit can occur in the intended place. +// Thread.currentThread().interrupt(); +// } +// } +// } +// +// /** +// * Check if the file is a special file that we should skip +// * +// * @param processTask a task whose file to check if should be queued of +// * skipped +// * @return true if should be enqueued, false otherwise +// */ +// private static boolean shouldEnqueueTask(final FileIngestTask processTask) { +// final AbstractFile aFile = processTask.getFile(); +// //if it's unalloc file, skip if so scheduled +// if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { +// return false; +// } +// String fileName = aFile.getName(); +// if (fileName.equals(".") || fileName.equals("..")) { +// return false; +// } else if (aFile instanceof org.sleuthkit.datamodel.File) { +// final org.sleuthkit.datamodel.File f = (File) aFile; +// //skip files in root dir, starting with $, containing : (not default attributes) +// //with meta address < 32, i.e. some special large NTFS and FAT files +// FileSystem fs = null; +// try { +// fs = f.getFileSystem(); +// } catch (TskCoreException ex) { +// logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS +// } +// TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP; +// if (fs != null) { +// fsType = fs.getFsType(); +// } +// if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { +// //not fat or ntfs, accept all files +// return true; +// } +// boolean isInRootDir = false; +// try { +// isInRootDir = f.getParentDirectory().isRoot(); +// } catch (TskCoreException ex) { +// logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS +// } +// if (isInRootDir && f.getMetaAddr() < 32) { +// String name = f.getName(); +// if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) { +// return false; +// } +// } else { +// return true; +// } +// } +// return true; +// } +// +// /** +// * Visitor that gets a collection of top level objects to be scheduled, such +// * as root directories (if there is FS) or LayoutFiles and virtual +// * directories, also if there is no FS. +// */ +// static class GetRootDirectoryVisitor extends GetFilesContentVisitor { +// +// @Override +// public Collection visit(VirtualDirectory ld) { +// //case when we hit a layout directoryor local file container, not under a real FS +// //or when root virt dir is scheduled +// Collection ret = new ArrayList<>(); +// ret.add(ld); +// return ret; +// } +// +// @Override +// public Collection visit(LayoutFile lf) { +// //case when we hit a layout file, not under a real FS +// Collection ret = new ArrayList<>(); +// ret.add(lf); +// return ret; +// } +// +// @Override +// public Collection visit(Directory drctr) { +// //we hit a real directory, a child of real FS +// Collection ret = new ArrayList<>(); +// ret.add(drctr); +// return ret; +// } +// +// @Override +// public Collection visit(FileSystem fs) { +// return getAllFromChildren(fs); +// } +// +// @Override +// public Collection visit(File file) { +// //can have derived files +// return getAllFromChildren(file); +// } +// +// @Override +// public Collection visit(DerivedFile derivedFile) { +// //can have derived files +// //TODO test this and overall scheduler with derived files +// return getAllFromChildren(derivedFile); +// } +// +// @Override +// public Collection visit(LocalFile localFile) { +// //can have local files +// //TODO test this and overall scheduler with local files +// return getAllFromChildren(localFile); +// } +// } +// +// /** +// * Root directory sorter +// */ +// private static class RootDirectoryTaskComparator implements Comparator { +// +// @Override +// public int compare(FileIngestTask q1, FileIngestTask q2) { +// AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile()); +// AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile()); +// if (p1 == p2) { +// return (int) (q2.getFile().getId() - q1.getFile().getId()); +// } else { +// return p2.ordinal() - p1.ordinal(); +// } +// } +// +// /** +// * Priority determination for sorted AbstractFile, used by +// * RootDirComparator +// */ +// private static class AbstractFilePriority { +// +// enum Priority { +// +// LAST, LOW, MEDIUM, HIGH +// } +// static final List LAST_PRI_PATHS = new ArrayList<>(); +// static final List LOW_PRI_PATHS = new ArrayList<>(); +// static final List MEDIUM_PRI_PATHS = new ArrayList<>(); +// static final List HIGH_PRI_PATHS = new ArrayList<>(); +// /* prioritize root directory folders based on the assumption that we are +// * looking for user content. Other types of investigations may want different +// * priorities. */ +// +// static /* prioritize root directory folders based on the assumption that we are +// * looking for user content. Other types of investigations may want different +// * priorities. */ { +// // these files have no structure, so they go last +// //unalloc files are handled as virtual files in getPriority() +// //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); +// //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE)); +// LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE)); +// LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE)); +// // orphan files are often corrupt and windows does not typically have +// // user content, so put them towards the bottom +// LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE)); +// LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE)); +// // all other files go into the medium category too +// MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE)); +// // user content is top priority +// HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE)); +// HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE)); +// HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE)); +// HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE)); +// } +// +// /** +// * Get the scheduling priority for a given file. +// * +// * @param abstractFile +// * @return +// */ +// static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) { +// if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { +// //quickly filter out unstructured content +// //non-fs virtual files and dirs, such as representing unalloc space +// return AbstractFilePriority.Priority.LAST; +// } +// //determine the fs files priority by name +// final String path = abstractFile.getName(); +// if (path == null) { +// return AbstractFilePriority.Priority.MEDIUM; +// } +// for (Pattern p : HIGH_PRI_PATHS) { +// Matcher m = p.matcher(path); +// if (m.find()) { +// return AbstractFilePriority.Priority.HIGH; +// } +// } +// for (Pattern p : MEDIUM_PRI_PATHS) { +// Matcher m = p.matcher(path); +// if (m.find()) { +// return AbstractFilePriority.Priority.MEDIUM; +// } +// } +// for (Pattern p : LOW_PRI_PATHS) { +// Matcher m = p.matcher(path); +// if (m.find()) { +// return AbstractFilePriority.Priority.LOW; +// } +// } +// for (Pattern p : LAST_PRI_PATHS) { +// Matcher m = p.matcher(path); +// if (m.find()) { +// return AbstractFilePriority.Priority.LAST; +// } +// } +// //default is medium +// return AbstractFilePriority.Priority.MEDIUM; +// } +// } +// } +//} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java index f8c280628d..b3bc34d1d5 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java @@ -38,7 +38,7 @@ import org.sleuthkit.datamodel.TskData; */ final class GetFilesCountVisitor extends ContentVisitor.Default { - private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); + private static final Logger logger = Logger.getLogger(GetFilesCountVisitor.class.getName()); @Override public Long visit(FileSystem fs) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 424aa638a2..529e0f5d1e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -40,7 +40,6 @@ import org.sleuthkit.datamodel.Content; final class IngestJob { private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); - private static final AtomicLong nextIngestJobId = new AtomicLong(0L); private static final ConcurrentHashMap ingestJobsById = new ConcurrentHashMap<>(); private final long id; private final Content rootDataSource; @@ -55,57 +54,7 @@ final class IngestJob { private ProgressHandle fileTasksProgress; private volatile boolean cancelled; - /** - * Creates an ingest job for a data source and starts up the ingest - * pipelines for the job. - * - * @param rootDataSource The data source to ingest. - * @param ingestModuleTemplates The ingest module templates to use to create - * the ingest pipelines for the job. - * @param processUnallocatedSpace Whether or not the job should include - * processing of unallocated space. - * @return A collection of ingest module startUp up errors, empty on - * success. - * @throws InterruptedException - */ - static List startJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { - long jobId = nextIngestJobId.incrementAndGet(); - IngestJob ingestJob = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); - List errors = ingestJob.startUp(); - if (errors.isEmpty()) { - ingestJobsById.put(jobId, ingestJob); - IngestManager.getInstance().fireIngestJobStarted(jobId); - } - return errors; - } - - static boolean jobsAreRunning() { - for (IngestJob job : ingestJobsById.values()) { - if (!job.isCancelled()) { - return true; - } - } - return false; - } - - static void addFileToJob(long ingestJobId, AbstractFile file) { // RJCTODO: Just one at a time? - IngestJob job = ingestJobsById.get(ingestJobId); - if (job != null) { -// long adjustedFilesCount = job.filesToIngestEstimate.incrementAndGet(); // RJCTODO: Not the best name now? -// job.fileTasksProgress.switchToIndeterminate(); // RJCTODO: Comment this stuff -// job.fileTasksProgress.switchToDeterminate((int) adjustedFilesCount); -// job.fileTasksProgress.progress(job.processedFiles.intValue()); - FileIngestTaskScheduler.getInstance().addTask(new FileIngestTask(job, file)); - } - } - - static void cancelAllJobs() { - for (IngestJob job : ingestJobsById.values()) { - job.cancel(); - } - } - - private IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { + IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { this.id = id; this.rootDataSource = dataSource; this.ingestModuleTemplates = ingestModuleTemplates; @@ -126,8 +75,6 @@ final class IngestJob { if (errors.isEmpty()) { startDataSourceIngestProgressBar(); startFileIngestProgressBar(); - FileIngestTaskScheduler.getInstance().addTasks(this, rootDataSource); // RJCTODO: Think about this ordering "solution" for small images - DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, rootDataSource)); } return errors; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java similarity index 99% rename from Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java rename to Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java index 23ab80d231..519df49999 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurator.java @@ -41,14 +41,14 @@ import org.sleuthkit.datamodel.Content; * for a particular context and for launching ingest jobs that process one or * more data sources using the ingest job configuration. */ -public final class IngestJobLauncher { +public final class IngestJobConfigurator { private static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; //NON-NLS private static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; //NON-NLS private static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; //NON-NLS private static final String MODULE_SETTINGS_FOLDER_PATH = new StringBuilder(PlatformUtil.getUserConfigDirectory()).append(File.separator).append("IngestModuleSettings").toString(); //NON-NLS private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS - private static final Logger logger = Logger.getLogger(IngestJobLauncher.class.getName()); + private static final Logger logger = Logger.getLogger(IngestJobConfigurator.class.getName()); private final String launcherContext; private String moduleSettingsFolderForContext = null; private final List warnings = new ArrayList<>(); @@ -61,7 +61,7 @@ public final class IngestJobLauncher { * * @param launcherContext The context identifier. */ - public IngestJobLauncher(String launcherContext) { + public IngestJobConfigurator(String launcherContext) { this.launcherContext = launcherContext; createModuleSettingsFolderForContext(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index b8d4493de4..968f72a7f8 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -60,7 +60,7 @@ public final class IngestJobContext { */ public void addFiles(List files) { for (AbstractFile file : files) { - IngestJob.addFileToJob(ingestJob.getId(), file); + IngestJobScheduler.getInstance().addFileToIngestJob(ingestJob, file); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java new file mode 100755 index 0000000000..0141f3587f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java @@ -0,0 +1,462 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012-2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DerivedFile; +import org.sleuthkit.datamodel.Directory; +import org.sleuthkit.datamodel.File; +import org.sleuthkit.datamodel.FileSystem; +import org.sleuthkit.datamodel.LayoutFile; +import org.sleuthkit.datamodel.LocalFile; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; +import org.sleuthkit.datamodel.VirtualDirectory; + +final class IngestJobScheduler { + + private static final Logger logger = Logger.getLogger(IngestJobScheduler.class.getName()); + private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); + private static IngestJobScheduler instance = new IngestJobScheduler(); + private final AtomicLong nextIngestJobId = new AtomicLong(0L); + private final ConcurrentHashMap ingestJobsById = new ConcurrentHashMap<>(); + private final LinkedBlockingQueue dataSourceTasks = new LinkedBlockingQueue<>(); + private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); + private final List directoryTasks = new ArrayList<>(); + private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); + private final DataSourceIngestTaskQueue dataSourceIngestTaskQueue = new DataSourceIngestTaskQueue(); + private final FileIngestTaskQueue fileIngestTaskQueue = new FileIngestTaskQueue(); + + static IngestJobScheduler getInstance() { + return instance; + } + + private IngestJobScheduler() { + } + + /** + * Creates an ingest job for a data source. + * + * @param rootDataSource The data source to ingest. + * @param ingestModuleTemplates The ingest module templates to use to create + * the ingest pipelines for the job. + * @param processUnallocatedSpace Whether or not the job should include + * processing of unallocated space. + * @return A collection of ingest module start up errors, empty on success. + * @throws InterruptedException + */ + synchronized List startIngestJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { + long jobId = nextIngestJobId.incrementAndGet(); + IngestJob job = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); + ingestJobsById.put(jobId, job); + IngestManager.getInstance().fireIngestJobStarted(jobId); + List errors = job.startUp(); + if (errors.isEmpty()) { + addDataSourceToIngestJob(job, dataSource); + } else { + ingestJobsById.remove(jobId); + IngestManager.getInstance().fireIngestJobCancelled(jobId); + } + return errors; + } + + boolean ingestJobsAreRunning() { + for (IngestJob job : ingestJobsById.values()) { + if (!job.isCancelled()) { + return true; + } + } + return false; + } + + synchronized void addDataSourceToIngestJob(IngestJob job, Content dataSource) throws InterruptedException { + // If the thread executing this code is interrupted, it is because the + // the number of ingest threads has been decreased while ingest jobs are + // running. The calling thread will exit in an orderly fashion, but the + // task still needs to be enqueued rather than lost. + DataSourceIngestTask task = new DataSourceIngestTask(job, dataSource); + while (true) { + try { + dataSourceTasks.put(task); + break; + } catch (InterruptedException ex) { + // Reset the interrupted status of the thread so the orderly + // exit can occur in the intended place. + Thread.currentThread().interrupt(); + } + } + + Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); + List firstLevelFiles = new ArrayList<>(); + if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { + // The data source is file. + firstLevelFiles.add((AbstractFile) dataSource); + } else { + for (AbstractFile root : rootObjects) { + List children; + try { + children = root.getChildren(); + if (children.isEmpty()) { + //add the root itself, could be unalloc file, child of volume or image + firstLevelFiles.add(root); + } else { + //root for fs root dir, schedule children dirs/files + for (Content child : children) { + if (child instanceof AbstractFile) { + firstLevelFiles.add((AbstractFile) child); + } + } + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS + } + } + } + for (AbstractFile firstLevelFile : firstLevelFiles) { + FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); + if (shouldEnqueueFileTask(fileTask)) { + rootDirectoryTasks.add(fileTask); + fileTask.getIngestJob().notifyTaskAdded(); + } + } + + // Reshuffle/update the dir and file level queues if needed + updateFileTaskQueues(); + } + + synchronized void addFileToIngestJob(IngestJob job, AbstractFile file) { // RJCTODO: Just one at a time? + FileIngestTask task = new FileIngestTask(job, file); + if (shouldEnqueueFileTask(task)) { + addTaskToFileQueue(task); + } + } + + private synchronized void updateFileTaskQueues() throws InterruptedException { + // we loop because we could have a directory that has all files + // that do not get enqueued + while (true) { + // There are files in the queue, we're done + if (fileTasks.isEmpty() == false) { + return; + } + // fill in the directory queue if it is empty. + if (this.directoryTasks.isEmpty()) { + // bail out if root is also empty -- we are done + if (rootDirectoryTasks.isEmpty()) { + return; + } + FileIngestTask rootTask = rootDirectoryTasks.pollFirst(); + directoryTasks.add(rootTask); + } + //pop and push AbstractFile directory children if any + //add the popped and its leaf children onto cur file list + FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); + final AbstractFile parentFile = parentTask.getFile(); + // add itself to the file list + if (shouldEnqueueFileTask(parentTask)) { + addTaskToFileQueue(parentTask); + } + // add its children to the file and directory lists + try { + List children = parentFile.getChildren(); + for (Content c : children) { + if (c instanceof AbstractFile) { + AbstractFile childFile = (AbstractFile) c; + FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); + if (childFile.hasChildren()) { + directoryTasks.add(childTask); + childTask.getIngestJob().notifyTaskAdded(); + } else if (shouldEnqueueFileTask(childTask)) { + addTaskToFileQueue(childTask); + } + } + } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex); + } + } + } + + private void addTaskToFileQueue(FileIngestTask task) { + // If the thread executing this code is interrupted, it is because the + // the number of ingest threads has been decreased while ingest jobs are + // running. The calling thread will exit in an orderly fashion, but the + // task still needs to be enqueued rather than lost. + while (true) { + try { + fileTasks.put(task); + break; + } catch (InterruptedException ex) { + // Reset the interrupted status of the thread so the orderly + // exit can occur in the intended place. + Thread.currentThread().interrupt(); + } + } + } + + /** + * Check if the file is a special file that we should skip + * + * @param processTask a task whose file to check if should be queued of + * skipped + * @return true if should be enqueued, false otherwise + */ + private static boolean shouldEnqueueFileTask(final FileIngestTask processTask) { + final AbstractFile aFile = processTask.getFile(); + //if it's unalloc file, skip if so scheduled + if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { + return false; + } + String fileName = aFile.getName(); + if (fileName.equals(".") || fileName.equals("..")) { + return false; + } else if (aFile instanceof org.sleuthkit.datamodel.File) { + final org.sleuthkit.datamodel.File f = (File) aFile; + //skip files in root dir, starting with $, containing : (not default attributes) + //with meta address < 32, i.e. some special large NTFS and FAT files + FileSystem fs = null; + try { + fs = f.getFileSystem(); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS + } + TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP; + if (fs != null) { + fsType = fs.getFsType(); + } + if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { + //not fat or ntfs, accept all files + return true; + } + boolean isInRootDir = false; + try { + isInRootDir = f.getParentDirectory().isRoot(); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS + } + if (isInRootDir && f.getMetaAddr() < 32) { + String name = f.getName(); + if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) { + return false; + } + } else { + return true; + } + } + return true; + } + + void cancelAllIngestJobs() { + for (IngestJob job : ingestJobsById.values()) { + job.cancel(); + } + } + + IngestTaskQueue getDataSourceIngestTaskQueue() { + return dataSourceIngestTaskQueue; + } + + IngestTaskQueue getFileIngestTaskQueue() { + return fileIngestTaskQueue; + } + + /** + * Finds top level objects such as file system root directories, layout + * files and virtual directories. + */ + private static class GetRootDirectoryVisitor extends GetFilesContentVisitor { + + @Override + public Collection visit(VirtualDirectory ld) { + //case when we hit a layout directoryor local file container, not under a real FS + //or when root virt dir is scheduled + Collection ret = new ArrayList<>(); + ret.add(ld); + return ret; + } + + @Override + public Collection visit(LayoutFile lf) { + //case when we hit a layout file, not under a real FS + Collection ret = new ArrayList<>(); + ret.add(lf); + return ret; + } + + @Override + public Collection visit(Directory drctr) { + //we hit a real directory, a child of real FS + Collection ret = new ArrayList<>(); + ret.add(drctr); + return ret; + } + + @Override + public Collection visit(FileSystem fs) { + return getAllFromChildren(fs); + } + + @Override + public Collection visit(File file) { + //can have derived files + return getAllFromChildren(file); + } + + @Override + public Collection visit(DerivedFile derivedFile) { + //can have derived files + //TODO test this and overall scheduler with derived files + return getAllFromChildren(derivedFile); + } + + @Override + public Collection visit(LocalFile localFile) { + //can have local files + //TODO test this and overall scheduler with local files + return getAllFromChildren(localFile); + } + } + + private static class RootDirectoryTaskComparator implements Comparator { + + @Override + public int compare(FileIngestTask q1, FileIngestTask q2) { + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile()); + AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile()); + if (p1 == p2) { + return (int) (q2.getFile().getId() - q1.getFile().getId()); + } else { + return p2.ordinal() - p1.ordinal(); + } + } + + private static class AbstractFilePriority { + + enum Priority { + + LAST, LOW, MEDIUM, HIGH + } + static final List LAST_PRI_PATHS = new ArrayList<>(); + static final List LOW_PRI_PATHS = new ArrayList<>(); + static final List MEDIUM_PRI_PATHS = new ArrayList<>(); + static final List HIGH_PRI_PATHS = new ArrayList<>(); + /* prioritize root directory folders based on the assumption that we are + * looking for user content. Other types of investigations may want different + * priorities. */ + + static /* prioritize root directory folders based on the assumption that we are + * looking for user content. Other types of investigations may want different + * priorities. */ { + // these files have no structure, so they go last + //unalloc files are handled as virtual files in getPriority() + //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); + //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE)); + LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE)); + LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE)); + // orphan files are often corrupt and windows does not typically have + // user content, so put them towards the bottom + LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE)); + LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE)); + // all other files go into the medium category too + MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE)); + // user content is top priority + HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE)); + HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE)); + HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE)); + HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE)); + } + + /** + * Get the scheduling priority for a given file. + * + * @param abstractFile + * @return + */ + static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) { + if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { + //quickly filter out unstructured content + //non-fs virtual files and dirs, such as representing unalloc space + return AbstractFilePriority.Priority.LAST; + } + //determine the fs files priority by name + final String path = abstractFile.getName(); + if (path == null) { + return AbstractFilePriority.Priority.MEDIUM; + } + for (Pattern p : HIGH_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.HIGH; + } + } + for (Pattern p : MEDIUM_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.MEDIUM; + } + } + for (Pattern p : LOW_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.LOW; + } + } + for (Pattern p : LAST_PRI_PATHS) { + Matcher m = p.matcher(path); + if (m.find()) { + return AbstractFilePriority.Priority.LAST; + } + } + //default is medium + return AbstractFilePriority.Priority.MEDIUM; + } + } + } + + private class DataSourceIngestTaskQueue implements IngestTaskQueue { + + @Override + public IngestTask getNextTask() throws InterruptedException { + return dataSourceTasks.take(); + } + } + + private class FileIngestTaskQueue implements IngestTaskQueue { + + @Override + public IngestTask getNextTask() throws InterruptedException { + FileIngestTask task = fileTasks.take(); + updateFileTaskQueues(); + return task; + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 36cc73fefb..d09783f0d1 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -55,6 +55,7 @@ public class IngestManager { private static final IngestManager instance = new IngestManager(); private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); + private final IngestJobScheduler scheduler = IngestJobScheduler.getInstance(); private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); @@ -157,7 +158,7 @@ public class IngestManager { */ private void startDataSourceIngestThread() { long threadId = nextThreadId.incrementAndGet(); - Future handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread()); + Future handle = dataSourceIngestThreadPool.submit(new ExecuteIngestTasksThread(scheduler.getDataSourceIngestTaskQueue())); dataSourceIngestThreads.put(threadId, handle); } @@ -167,7 +168,7 @@ public class IngestManager { */ private void startFileIngestThread() { long threadId = nextThreadId.incrementAndGet(); - Future handle = fileIngestThreadPool.submit(new FileIngestThread()); + Future handle = fileIngestThreadPool.submit(new ExecuteIngestTasksThread(scheduler.getFileIngestTaskQueue())); fileIngestThreads.put(threadId, handle); } @@ -199,7 +200,7 @@ public class IngestManager { * @return True if any ingest jobs are in progress, false otherwise. */ public boolean isIngestRunning() { - return IngestJob.jobsAreRunning(); + return scheduler.ingestJobsAreRunning(); } public void cancelAllIngestJobs() { @@ -221,7 +222,7 @@ public class IngestManager { // Cancel all the jobs already created. This will make the the ingest // threads flush out any lingering ingest tasks without processing them. - IngestJob.cancelAllJobs(); + scheduler.cancelAllIngestJobs(); } /** @@ -406,8 +407,7 @@ public class IngestManager { } /** - * A Callable that creates ingest jobs and submits the initial data source - * and file ingest tasks to the task schedulers. + * Creates ingest jobs. */ private class StartIngestJobsThread implements Callable { @@ -454,7 +454,7 @@ public class IngestManager { } // Start an ingest job for the data source. - List errors = IngestJob.startJob(dataSource, moduleTemplates, processUnallocatedSpace); + List errors = scheduler.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace); if (!errors.isEmpty()) { // Report the errors to the user. They have already been logged. StringBuilder moduleStartUpErrors = new StringBuilder(); @@ -495,17 +495,21 @@ public class IngestManager { } /** - * A Runnable that acts as a consumer for the data ingest task scheduler's - * task queue. + * A consumer for an ingest task queue. */ - private class DataSourceIngestThread implements Runnable { + private class ExecuteIngestTasksThread implements Runnable { + + private IngestTaskQueue tasks; + + ExecuteIngestTasksThread(IngestTaskQueue tasks) { + this.tasks = tasks; + } @Override public void run() { - DataSourceIngestTaskScheduler scheduler = DataSourceIngestTaskScheduler.getInstance(); while (true) { try { - DataSourceIngestTask task = scheduler.getNextTask(); // Blocks. + IngestTask task = tasks.getNextTask(); // Blocks. task.execute(); } catch (InterruptedException ex) { break; @@ -518,31 +522,7 @@ public class IngestManager { } /** - * A Runnable that acts as a consumer for the file task scheduler's task - * queue. - */ - private static class FileIngestThread implements Runnable { - - @Override - public void run() { - FileIngestTaskScheduler scheduler = FileIngestTaskScheduler.getInstance(); - while (true) { - try { - FileIngestTask task = scheduler.getNextTask(); // Blocks. - task.execute(); - } catch (InterruptedException ex) { - break; - } - if (Thread.currentThread().isInterrupted()) { - break; - } - } - } - } - - /** - * A Runnable that fires ingest events to ingest manager property change - * listeners. + * Fires ingest events to ingest manager property change listeners. */ private static class FireIngestEventThread implements Runnable { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskQueue.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskQueue.java new file mode 100755 index 0000000000..d18f7047b7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskQueue.java @@ -0,0 +1,23 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +interface IngestTaskQueue { + IngestTask getNextTask() throws InterruptedException; +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java index a31b92b27d..9f722789d2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/RunIngestModulesDialog.java @@ -45,11 +45,11 @@ public final class RunIngestModulesDialog extends JDialog { private static final String TITLE = NbBundle.getMessage(RunIngestModulesDialog.class, "IngestDialog.title.text"); private static Dimension DIMENSIONS = new Dimension(500, 300); private List dataSources = new ArrayList<>(); - private IngestJobLauncher ingestJobLauncher; + private IngestJobConfigurator ingestJobLauncher; public RunIngestModulesDialog(JFrame frame, String title, boolean modal) { super(frame, title, modal); - ingestJobLauncher = new IngestJobLauncher(RunIngestModulesDialog.class.getCanonicalName()); + ingestJobLauncher = new IngestJobConfigurator(RunIngestModulesDialog.class.getCanonicalName()); List messages = ingestJobLauncher.getIngestJobConfigWarnings(); if (messages.isEmpty() == false) { StringBuilder warning = new StringBuilder(); diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java index b7099c5098..81189cc690 100755 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java @@ -32,7 +32,7 @@ import org.openide.util.NbBundle; import org.openide.util.Utilities; import org.openide.util.Lookup; import org.openide.util.actions.Presenter; -import org.sleuthkit.autopsy.ingest.IngestJobLauncher; +import org.sleuthkit.autopsy.ingest.IngestJobConfigurator; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; From c900f0f95a6f7eb43e6c31428f2e8cdaeea3e840 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 2 May 2014 15:03:46 -0400 Subject: [PATCH 5/5] Complete improved ingest framework --- .../autopsy/corecomponents/GeneralPanel.form | 11 +- .../autopsy/corecomponents/GeneralPanel.java | 60 ++- .../autopsy/ingest/DataSourceIngestTask.java | 17 +- .../ingest/DataSourceIngestTaskScheduler.java | 56 --- .../autopsy/ingest/FileIngestTask.java | 23 +- .../ingest/FileIngestTaskScheduler.java | 393 ------------------ .../ingest/GetRootDirectoryVisitor.java | 88 ++++ .../sleuthkit/autopsy/ingest/IngestJob.java | 80 ++-- .../autopsy/ingest/IngestJobContext.java | 2 +- .../autopsy/ingest/IngestManager.java | 12 +- ...JobScheduler.java => IngestScheduler.java} | 166 ++++---- .../sleuthkit/autopsy/ingest/IngestTask.java | 15 +- 12 files changed, 262 insertions(+), 661 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java delete mode 100755 Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/GetRootDirectoryVisitor.java rename Core/src/org/sleuthkit/autopsy/ingest/{IngestJobScheduler.java => IngestScheduler.java} (79%) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.form index ea8c8fed21..2c0bbd1ee6 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.form @@ -188,15 +188,12 @@ - - - - - - + - + + + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.java index ea334dae46..d34a49b16a 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/GeneralPanel.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.corecomponents; import java.util.prefs.Preferences; +import javax.swing.DefaultComboBoxModel; import org.openide.util.NbPreferences; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.IngestManager; @@ -33,10 +34,35 @@ final class GeneralPanel extends javax.swing.JPanel { GeneralPanel(GeneralOptionsPanelController controller) { initComponents(); + numberOfFileIngestThreadsComboBox.setModel(new DefaultComboBoxModel<>(new Integer[]{1, 2, 4, 8, 16})); ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); // TODO listen to changes in form fields and call controller.changed() } + void load() { + boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false); + keepCurrentViewerRB.setSelected(keepPreferredViewer); + useBestViewerRB.setSelected(!keepPreferredViewer); + boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true); + useLocalTimeRB.setSelected(useLocalTime); + useGMTTimeRB.setSelected(!useLocalTime); + dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false)); + viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true)); + numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getNumberOfFileIngestThreads()); + } + + void store() { + prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected()); + prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected()); + prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected()); + prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected()); + IngestManager.setNumberOfFileIngestThreads((Integer) numberOfFileIngestThreadsComboBox.getSelectedItem()); + } + + boolean valid() { + return true; + } + /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always @@ -57,7 +83,7 @@ final class GeneralPanel extends javax.swing.JPanel { dataSourcesHideKnownCB = new javax.swing.JCheckBox(); viewsHideKnownCB = new javax.swing.JCheckBox(); jLabel4 = new javax.swing.JLabel(); - numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox(); + numberOfFileIngestThreadsComboBox = new javax.swing.JComboBox(); buttonGroup1.add(useBestViewerRB); useBestViewerRB.setSelected(true); @@ -97,9 +123,6 @@ final class GeneralPanel extends javax.swing.JPanel { org.openide.awt.Mnemonics.setLocalizedText(jLabel4, org.openide.util.NbBundle.getMessage(GeneralPanel.class, "GeneralPanel.jLabel4.text")); // NOI18N - numberOfFileIngestThreadsComboBox.setModel(new javax.swing.DefaultComboBoxModel(new String[] { "1", "2", "3", "4" })); - numberOfFileIngestThreadsComboBox.setSelectedIndex(1); - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -163,33 +186,8 @@ final class GeneralPanel extends javax.swing.JPanel { }//GEN-LAST:event_useBestViewerRBActionPerformed private void useGMTTimeRBActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRBActionPerformed - ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); + ContentUtils.setDisplayInLocalTime(useLocalTimeRB.isSelected()); }//GEN-LAST:event_useGMTTimeRBActionPerformed - - void load() { - boolean keepPreferredViewer = prefs.getBoolean(KEEP_PREFERRED_VIEWER, false); - keepCurrentViewerRB.setSelected(keepPreferredViewer); - useBestViewerRB.setSelected(!keepPreferredViewer); - boolean useLocalTime = prefs.getBoolean(USE_LOCAL_TIME, true); - useLocalTimeRB.setSelected(useLocalTime); - useGMTTimeRB.setSelected(!useLocalTime); - dataSourcesHideKnownCB.setSelected(prefs.getBoolean(DS_HIDE_KNOWN, false)); - viewsHideKnownCB.setSelected(prefs.getBoolean(VIEWS_HIDE_KNOWN, true)); - numberOfFileIngestThreadsComboBox.setSelectedItem(IngestManager.getInstance().getNumberOfFileIngestThreads()); - } - - void store() { - prefs.putBoolean(KEEP_PREFERRED_VIEWER, keepCurrentViewerRB.isSelected()); - prefs.putBoolean(USE_LOCAL_TIME, useLocalTimeRB.isSelected()); - prefs.putBoolean(DS_HIDE_KNOWN, dataSourcesHideKnownCB.isSelected()); - prefs.putBoolean(VIEWS_HIDE_KNOWN, viewsHideKnownCB.isSelected()); - IngestManager.getInstance().setNumberOfFileIngestThreads(Integer.valueOf(numberOfFileIngestThreadsComboBox.getSelectedItem().toString())); - } - - boolean valid() { - // TODO check whether form is consistent and complete - return true; - } // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.ButtonGroup buttonGroup1; private javax.swing.ButtonGroup buttonGroup3; @@ -199,7 +197,7 @@ final class GeneralPanel extends javax.swing.JPanel { private javax.swing.JLabel jLabel3; private javax.swing.JLabel jLabel4; private javax.swing.JRadioButton keepCurrentViewerRB; - private javax.swing.JComboBox numberOfFileIngestThreadsComboBox; + private javax.swing.JComboBox numberOfFileIngestThreadsComboBox; private javax.swing.JRadioButton useBestViewerRB; private javax.swing.JRadioButton useGMTTimeRB; private javax.swing.JRadioButton useLocalTimeRB; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index c62b6796a1..5598033397 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2012-2014 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,26 +20,21 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.Content; -final class DataSourceIngestTask implements IngestTask { +final class DataSourceIngestTask extends IngestTask { - private final IngestJob ingestJob; private final Content dataSource; DataSourceIngestTask(IngestJob ingestJob, Content dataSource) { - this.ingestJob = ingestJob; + super(ingestJob); this.dataSource = dataSource; } - - IngestJob getIngestJob() { - return ingestJob; - } - + Content getDataSource() { return dataSource; } @Override - public void execute() throws InterruptedException { - ingestJob.process(dataSource); + void execute() throws InterruptedException { + getIngestJob().process(dataSource); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java deleted file mode 100755 index ba89a669a4..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2012-2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import java.util.concurrent.LinkedBlockingQueue; - -//final class DataSourceIngestTaskScheduler implements IngestTaskQueue { -// -// private static DataSourceIngestTaskScheduler instance = new DataSourceIngestTaskScheduler(); -// private final LinkedBlockingQueue tasks = new LinkedBlockingQueue<>(); -// -// static DataSourceIngestTaskScheduler getInstance() { -// return instance; -// } -// -// private DataSourceIngestTaskScheduler() { -// } -// -// void addTask(DataSourceIngestTask task) { -// // If the thread executing this code is interrupted, it is because the -// // number of ingest threads has been decreased while ingest jobs are -// // running. This thread will exit in an orderly fashion, but the task -// // still needs to be enqueued rather than lost. -// while (true) { -// try { -// tasks.put(task); -// break; -// } catch (InterruptedException ex) { -// // Reset the interrupted status of the thread so the orderly -// // exit can occur in the intended place. -// Thread.currentThread().interrupt(); -// } -// } -// } -// -// @Override -// public DataSourceIngestTask getNextTask() throws InterruptedException { -// return tasks.take(); -// } -//} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 75e9165ded..3134b6deb6 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -21,27 +21,22 @@ package org.sleuthkit.autopsy.ingest; import java.util.Objects; import org.sleuthkit.datamodel.AbstractFile; -final class FileIngestTask implements IngestTask { +final class FileIngestTask extends IngestTask { - private final IngestJob ingestJob; private final AbstractFile file; - FileIngestTask(IngestJob task, AbstractFile file) { - this.ingestJob = task; + FileIngestTask(IngestJob job, AbstractFile file) { + super(job); this.file = file; } - public IngestJob getIngestJob() { // RJCTODO: Maybe add to interface - return ingestJob; - } - - public AbstractFile getFile() { + AbstractFile getFile() { return file; } @Override - public void execute() throws InterruptedException { - ingestJob.process(file); + void execute() throws InterruptedException { + getIngestJob().process(file); } @Override @@ -53,7 +48,9 @@ final class FileIngestTask implements IngestTask { return false; } FileIngestTask other = (FileIngestTask) obj; - if (this.ingestJob != other.ingestJob && (this.ingestJob == null || !this.ingestJob.equals(other.ingestJob))) { + IngestJob job = getIngestJob(); + IngestJob otherJob = other.getIngestJob(); + if (job != otherJob && (job == null || !job.equals(otherJob))) { return false; } if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { @@ -65,7 +62,7 @@ final class FileIngestTask implements IngestTask { @Override public int hashCode() { int hash = 5; - hash = 47 * hash + Objects.hashCode(this.ingestJob); + hash = 47 * hash + Objects.hashCode(getIngestJob()); hash = 47 * hash + Objects.hashCode(this.file); return hash; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java deleted file mode 100755 index c0a4b34295..0000000000 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2012-2014 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.ingest; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.TreeSet; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DerivedFile; -import org.sleuthkit.datamodel.Directory; -import org.sleuthkit.datamodel.File; -import org.sleuthkit.datamodel.FileSystem; -import org.sleuthkit.datamodel.LayoutFile; -import org.sleuthkit.datamodel.LocalFile; -import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.VirtualDirectory; - -//final class FileIngestTaskScheduler implements IngestTaskQueue { -// -// private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); -// private static FileIngestTaskScheduler instance = new FileIngestTaskScheduler(); -// private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); -// private final List directoryTasks = new ArrayList<>(); -// private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); -// private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); -// -// static FileIngestTaskScheduler getInstance() { -// return instance; -// } -// -// private FileIngestTaskScheduler() { -// } -// -// synchronized void addTasks(IngestJob job, Content dataSource) throws InterruptedException { -// Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); -// List firstLevelFiles = new ArrayList<>(); -// if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { -// // The data source is file. -// firstLevelFiles.add((AbstractFile) dataSource); -// } else { -// for (AbstractFile root : rootObjects) { -// List children; -// try { -// children = root.getChildren(); -// if (children.isEmpty()) { -// //add the root itself, could be unalloc file, child of volume or image -// firstLevelFiles.add(root); -// } else { -// //root for fs root dir, schedule children dirs/files -// for (Content child : children) { -// if (child instanceof AbstractFile) { -// firstLevelFiles.add((AbstractFile) child); -// } -// } -// } -// } catch (TskCoreException ex) { -// logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS -// } -// } -// } -// for (AbstractFile firstLevelFile : firstLevelFiles) { -// FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); -// if (shouldEnqueueTask(fileTask)) { -// rootDirectoryTasks.add(fileTask); -// fileTask.getIngestJob().notifyTaskAdded(); -// } -// } -// -// // Reshuffle/update the dir and file level queues if needed -// updateQueues(); -// } -// -// synchronized void addTask(FileIngestTask task) { -// if (shouldEnqueueTask(task)) { -// addTaskToFileQueue(task, true); -// } -// } -// -// @Override -// public FileIngestTask getNextTask() throws InterruptedException { -// FileIngestTask task = fileTasks.take(); -// updateQueues(); -// return task; -// } -// -// private void updateQueues() throws InterruptedException { -// // we loop because we could have a directory that has all files -// // that do not get enqueued -// while (true) { -// // There are files in the queue, we're done -// if (fileTasks.isEmpty() == false) { -// return; -// } -// // fill in the directory queue if it is empty. -// if (this.directoryTasks.isEmpty()) { -// // bail out if root is also empty -- we are done -// if (rootDirectoryTasks.isEmpty()) { -// return; -// } -// FileIngestTask rootTask = rootDirectoryTasks.pollFirst(); -// directoryTasks.add(rootTask); -// } -// //pop and push AbstractFile directory children if any -// //add the popped and its leaf children onto cur file list -// FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); -// final AbstractFile parentFile = parentTask.getFile(); -// // add itself to the file list -// if (shouldEnqueueTask(parentTask)) { -// addTaskToFileQueue(parentTask, false); -// } -// // add its children to the file and directory lists -// try { -// List children = parentFile.getChildren(); -// for (Content c : children) { -// if (c instanceof AbstractFile) { -// AbstractFile childFile = (AbstractFile) c; -// FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); -// if (childFile.hasChildren()) { -// directoryTasks.add(childTask); -// childTask.getIngestJob().notifyTaskAdded(); -// } else if (shouldEnqueueTask(childTask)) { -// addTaskToFileQueue(childTask, true); -// } -// } -// } -// } catch (TskCoreException ex) { -// logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex); -// } -// } -// } -// -// private void addTaskToFileQueue(FileIngestTask task, boolean isNewTask) { -// if (isNewTask) { -// // The capacity of the file tasks queue is not bounded, so the call -// // to put() should not block except for normal synchronized access. -// // Still, notify the job that the task has been added first so that -// // the take() of the task cannot occur before the notification. -// task.getIngestJob().notifyTaskAdded(); -// } -// // If the thread executing this code is ever interrupted, it is -// // because the number of ingest threads has been decreased while -// // ingest jobs are running. This thread will exit in an orderly fashion, -// // but the task still needs to be enqueued rather than lost. -// while (true) { -// try { -// fileTasks.put(task); -// break; -// } catch (InterruptedException ex) { -// // Reset the interrupted status of the thread so the orderly -// // exit can occur in the intended place. -// Thread.currentThread().interrupt(); -// } -// } -// } -// -// /** -// * Check if the file is a special file that we should skip -// * -// * @param processTask a task whose file to check if should be queued of -// * skipped -// * @return true if should be enqueued, false otherwise -// */ -// private static boolean shouldEnqueueTask(final FileIngestTask processTask) { -// final AbstractFile aFile = processTask.getFile(); -// //if it's unalloc file, skip if so scheduled -// if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { -// return false; -// } -// String fileName = aFile.getName(); -// if (fileName.equals(".") || fileName.equals("..")) { -// return false; -// } else if (aFile instanceof org.sleuthkit.datamodel.File) { -// final org.sleuthkit.datamodel.File f = (File) aFile; -// //skip files in root dir, starting with $, containing : (not default attributes) -// //with meta address < 32, i.e. some special large NTFS and FAT files -// FileSystem fs = null; -// try { -// fs = f.getFileSystem(); -// } catch (TskCoreException ex) { -// logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS -// } -// TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP; -// if (fs != null) { -// fsType = fs.getFsType(); -// } -// if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) { -// //not fat or ntfs, accept all files -// return true; -// } -// boolean isInRootDir = false; -// try { -// isInRootDir = f.getParentDirectory().isRoot(); -// } catch (TskCoreException ex) { -// logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS -// } -// if (isInRootDir && f.getMetaAddr() < 32) { -// String name = f.getName(); -// if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) { -// return false; -// } -// } else { -// return true; -// } -// } -// return true; -// } -// -// /** -// * Visitor that gets a collection of top level objects to be scheduled, such -// * as root directories (if there is FS) or LayoutFiles and virtual -// * directories, also if there is no FS. -// */ -// static class GetRootDirectoryVisitor extends GetFilesContentVisitor { -// -// @Override -// public Collection visit(VirtualDirectory ld) { -// //case when we hit a layout directoryor local file container, not under a real FS -// //or when root virt dir is scheduled -// Collection ret = new ArrayList<>(); -// ret.add(ld); -// return ret; -// } -// -// @Override -// public Collection visit(LayoutFile lf) { -// //case when we hit a layout file, not under a real FS -// Collection ret = new ArrayList<>(); -// ret.add(lf); -// return ret; -// } -// -// @Override -// public Collection visit(Directory drctr) { -// //we hit a real directory, a child of real FS -// Collection ret = new ArrayList<>(); -// ret.add(drctr); -// return ret; -// } -// -// @Override -// public Collection visit(FileSystem fs) { -// return getAllFromChildren(fs); -// } -// -// @Override -// public Collection visit(File file) { -// //can have derived files -// return getAllFromChildren(file); -// } -// -// @Override -// public Collection visit(DerivedFile derivedFile) { -// //can have derived files -// //TODO test this and overall scheduler with derived files -// return getAllFromChildren(derivedFile); -// } -// -// @Override -// public Collection visit(LocalFile localFile) { -// //can have local files -// //TODO test this and overall scheduler with local files -// return getAllFromChildren(localFile); -// } -// } -// -// /** -// * Root directory sorter -// */ -// private static class RootDirectoryTaskComparator implements Comparator { -// -// @Override -// public int compare(FileIngestTask q1, FileIngestTask q2) { -// AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile()); -// AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile()); -// if (p1 == p2) { -// return (int) (q2.getFile().getId() - q1.getFile().getId()); -// } else { -// return p2.ordinal() - p1.ordinal(); -// } -// } -// -// /** -// * Priority determination for sorted AbstractFile, used by -// * RootDirComparator -// */ -// private static class AbstractFilePriority { -// -// enum Priority { -// -// LAST, LOW, MEDIUM, HIGH -// } -// static final List LAST_PRI_PATHS = new ArrayList<>(); -// static final List LOW_PRI_PATHS = new ArrayList<>(); -// static final List MEDIUM_PRI_PATHS = new ArrayList<>(); -// static final List HIGH_PRI_PATHS = new ArrayList<>(); -// /* prioritize root directory folders based on the assumption that we are -// * looking for user content. Other types of investigations may want different -// * priorities. */ -// -// static /* prioritize root directory folders based on the assumption that we are -// * looking for user content. Other types of investigations may want different -// * priorities. */ { -// // these files have no structure, so they go last -// //unalloc files are handled as virtual files in getPriority() -// //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE)); -// //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE)); -// LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE)); -// LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE)); -// // orphan files are often corrupt and windows does not typically have -// // user content, so put them towards the bottom -// LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE)); -// LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE)); -// // all other files go into the medium category too -// MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE)); -// // user content is top priority -// HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE)); -// HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE)); -// HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE)); -// HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE)); -// } -// -// /** -// * Get the scheduling priority for a given file. -// * -// * @param abstractFile -// * @return -// */ -// static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) { -// if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { -// //quickly filter out unstructured content -// //non-fs virtual files and dirs, such as representing unalloc space -// return AbstractFilePriority.Priority.LAST; -// } -// //determine the fs files priority by name -// final String path = abstractFile.getName(); -// if (path == null) { -// return AbstractFilePriority.Priority.MEDIUM; -// } -// for (Pattern p : HIGH_PRI_PATHS) { -// Matcher m = p.matcher(path); -// if (m.find()) { -// return AbstractFilePriority.Priority.HIGH; -// } -// } -// for (Pattern p : MEDIUM_PRI_PATHS) { -// Matcher m = p.matcher(path); -// if (m.find()) { -// return AbstractFilePriority.Priority.MEDIUM; -// } -// } -// for (Pattern p : LOW_PRI_PATHS) { -// Matcher m = p.matcher(path); -// if (m.find()) { -// return AbstractFilePriority.Priority.LOW; -// } -// } -// for (Pattern p : LAST_PRI_PATHS) { -// Matcher m = p.matcher(path); -// if (m.find()) { -// return AbstractFilePriority.Priority.LAST; -// } -// } -// //default is medium -// return AbstractFilePriority.Priority.MEDIUM; -// } -// } -// } -//} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GetRootDirectoryVisitor.java b/Core/src/org/sleuthkit/autopsy/ingest/GetRootDirectoryVisitor.java new file mode 100755 index 0000000000..a81ce17f6a --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/GetRootDirectoryVisitor.java @@ -0,0 +1,88 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2012 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.ArrayList; +import java.util.Collection; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.DerivedFile; +import org.sleuthkit.datamodel.Directory; +import org.sleuthkit.datamodel.File; +import org.sleuthkit.datamodel.FileSystem; +import org.sleuthkit.datamodel.LayoutFile; +import org.sleuthkit.datamodel.LocalFile; +import org.sleuthkit.datamodel.VirtualDirectory; + +/** + * Finds top level objects such as file system root directories, layout + * files and virtual directories. + */ +final class GetRootDirectoryVisitor extends GetFilesContentVisitor { + + @Override + public Collection visit(VirtualDirectory ld) { + //case when we hit a layout directoryor local file container, not under a real FS + //or when root virt dir is scheduled + Collection ret = new ArrayList<>(); + ret.add(ld); + return ret; + } + + @Override + public Collection visit(LayoutFile lf) { + //case when we hit a layout file, not under a real FS + Collection ret = new ArrayList<>(); + ret.add(lf); + return ret; + } + + @Override + public Collection visit(Directory drctr) { + //we hit a real directory, a child of real FS + Collection ret = new ArrayList<>(); + ret.add(drctr); + return ret; + } + + @Override + public Collection visit(FileSystem fs) { + return getAllFromChildren(fs); + } + + @Override + public Collection visit(File file) { + //can have derived files + return getAllFromChildren(file); + } + + @Override + public Collection visit(DerivedFile derivedFile) { + //can have derived files + //TODO test this and overall scheduler with derived files + return getAllFromChildren(derivedFile); + } + + @Override + public Collection visit(LocalFile localFile) { + //can have local files + //TODO test this and overall scheduler with local files + return getAllFromChildren(localFile); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 529e0f5d1e..22d32b3638 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -20,10 +20,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandleFactory; @@ -33,33 +30,26 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; -/** - * Encapsulates a data source and the ingest module pipelines to be used to - * ingest the data source. - */ final class IngestJob { private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); - private static final ConcurrentHashMap ingestJobsById = new ConcurrentHashMap<>(); private final long id; private final Content rootDataSource; private final List ingestModuleTemplates; private final boolean processUnallocatedSpace; private final LinkedBlockingQueue dataSourceIngestPipelines = new LinkedBlockingQueue<>(); private final LinkedBlockingQueue fileIngestPipelines = new LinkedBlockingQueue<>(); - private final AtomicInteger tasksInProgress = new AtomicInteger(0); - private final AtomicLong processedFiles = new AtomicLong(0L); - private final AtomicLong filesToIngestEstimate = new AtomicLong(0L); + private long estimatedFilesToProcess = 0L; // Guarded by this + private long processedFiles = 0L; // Guarded by this private ProgressHandle dataSourceTasksProgress; private ProgressHandle fileTasksProgress; - private volatile boolean cancelled; + private volatile boolean cancelled = false; IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { this.id = id; this.rootDataSource = dataSource; this.ingestModuleTemplates = ingestModuleTemplates; this.processUnallocatedSpace = processUnallocatedSpace; - this.cancelled = false; } long getId() { @@ -73,8 +63,8 @@ final class IngestJob { List startUp() throws InterruptedException { List errors = startUpIngestPipelines(); if (errors.isEmpty()) { - startDataSourceIngestProgressBar(); startFileIngestProgressBar(); + startDataSourceIngestProgressBar(); } return errors; } @@ -105,7 +95,8 @@ final class IngestJob { } } - return errors; + logIngestModuleErrors(errors); + return errors; // Returned so UI can report to user. } private void startDataSourceIngestProgressBar() { @@ -145,18 +136,9 @@ final class IngestJob { return true; } }); - long initialFilesCount = rootDataSource.accept(new GetFilesCountVisitor()); - filesToIngestEstimate.getAndAdd(initialFilesCount); + estimatedFilesToProcess = rootDataSource.accept(new GetFilesCountVisitor()); fileTasksProgress.start(); - fileTasksProgress.switchToDeterminate((int) initialFilesCount); // RJCTODO: This cast is troublesome, can use intValue - } - - /** - * Called by the ingest task schedulers when an ingest task is added to this - * ingest job. - */ - void notifyTaskAdded() { - tasksInProgress.incrementAndGet(); + fileTasksProgress.switchToDeterminate((int) estimatedFilesToProcess); } void process(Content dataSource) throws InterruptedException { @@ -172,7 +154,6 @@ final class IngestJob { } dataSourceIngestPipelines.put(pipeline); } - shutDownIfAllTasksCompleted(); } void process(AbstractFile file) throws InterruptedException { @@ -181,35 +162,37 @@ final class IngestJob { // shut down check needs to occur. if (!isCancelled()) { List errors = new ArrayList<>(); + synchronized (this) { + ++processedFiles; + if (processedFiles <= estimatedFilesToProcess) { + fileTasksProgress.progress(file.getName(), (int) processedFiles); + } else { + fileTasksProgress.progress(file.getName(), (int) estimatedFilesToProcess); + } + } FileIngestPipeline pipeline = fileIngestPipelines.take(); - fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); errors.addAll(pipeline.process(file)); fileIngestPipelines.put(pipeline); if (!errors.isEmpty()) { logIngestModuleErrors(errors); } } - shutDownIfAllTasksCompleted(); } - private void shutDownIfAllTasksCompleted() { - if (tasksInProgress.decrementAndGet() == 0) { - List errors = new ArrayList<>(); - while (!dataSourceIngestPipelines.isEmpty()) { - DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll(); - errors.addAll(pipeline.shutDown()); - } - while (!fileIngestPipelines.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelines.poll(); - errors.addAll(pipeline.shutDown()); - } - fileTasksProgress.finish(); - dataSourceTasksProgress.finish(); - ingestJobsById.remove(id); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - IngestManager.getInstance().fireIngestJobCompleted(id); + void shutDown() { + List errors = new ArrayList<>(); + while (!dataSourceIngestPipelines.isEmpty()) { + DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll(); + errors.addAll(pipeline.shutDown()); + } + while (!fileIngestPipelines.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelines.poll(); + errors.addAll(pipeline.shutDown()); + } + fileTasksProgress.finish(); + dataSourceTasksProgress.finish(); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); } } @@ -225,7 +208,8 @@ final class IngestJob { void cancel() { cancelled = true; - fileTasksProgress.finish(); // RJCTODO: What about the other progress bar? + fileTasksProgress.finish(); + dataSourceTasksProgress.finish(); IngestManager.getInstance().fireIngestJobCancelled(id); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 968f72a7f8..735f39bc28 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -60,7 +60,7 @@ public final class IngestJobContext { */ public void addFiles(List files) { for (AbstractFile file : files) { - IngestJobScheduler.getInstance().addFileToIngestJob(ingestJob, file); + IngestScheduler.getInstance().addFileToIngestJob(ingestJob, file); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index d09783f0d1..b5674ab698 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -48,23 +48,23 @@ public class IngestManager { private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1; private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; - private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4; + private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 16; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final IngestManager instance = new IngestManager(); private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); - private final IngestJobScheduler scheduler = IngestJobScheduler.getInstance(); + private final IngestScheduler scheduler = IngestScheduler.getInstance(); private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); + private final AtomicLong nextThreadId = new AtomicLong(0L); private final ConcurrentHashMap> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. - private final AtomicLong nextThreadId = new AtomicLong(0L); private volatile IngestMessageTopComponent ingestMessageBox; /** @@ -186,8 +186,8 @@ public class IngestManager { } long taskId = nextThreadId.incrementAndGet(); - Future task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); - fileIngestThreads.put(taskId, task); + Future task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); + startIngestJobThreads.put(taskId, task); if (ingestMessageBox != null) { ingestMessageBox.restoreMessages(); @@ -218,7 +218,6 @@ public class IngestManager { logger.log(Level.SEVERE, "Unexpected thread interrupt", ex); } } - startIngestJobThreads.clear(); // Make sure. // Cancel all the jobs already created. This will make the the ingest // threads flush out any lingering ingest tasks without processing them. @@ -511,6 +510,7 @@ public class IngestManager { try { IngestTask task = tasks.getNextTask(); // Blocks. task.execute(); + scheduler.ingestTaskIsCompleted(task); } catch (InterruptedException ex) { break; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java similarity index 79% rename from Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java rename to Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 0141f3587f..6bad7b6ab7 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -32,35 +32,31 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DerivedFile; -import org.sleuthkit.datamodel.Directory; import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.FileSystem; -import org.sleuthkit.datamodel.LayoutFile; -import org.sleuthkit.datamodel.LocalFile; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.VirtualDirectory; -final class IngestJobScheduler { +final class IngestScheduler { - private static final Logger logger = Logger.getLogger(IngestJobScheduler.class.getName()); + private static final IngestScheduler instance = new IngestScheduler(); + private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName()); private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); - private static IngestJobScheduler instance = new IngestJobScheduler(); - private final AtomicLong nextIngestJobId = new AtomicLong(0L); private final ConcurrentHashMap ingestJobsById = new ConcurrentHashMap<>(); private final LinkedBlockingQueue dataSourceTasks = new LinkedBlockingQueue<>(); - private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); - private final List directoryTasks = new ArrayList<>(); - private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); - private final DataSourceIngestTaskQueue dataSourceIngestTaskQueue = new DataSourceIngestTaskQueue(); - private final FileIngestTaskQueue fileIngestTaskQueue = new FileIngestTaskQueue(); + private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); // Guarded by this + private final List directoryTasks = new ArrayList<>(); // Guarded by this + private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); // Guarded by this + private final List tasksInProgress = new ArrayList<>(); // Guarded by this + private final DataSourceIngestTaskQueue dataSourceTaskDispenser = new DataSourceIngestTaskQueue(); + private final FileIngestTaskQueue fileTaskDispenser = new FileIngestTaskQueue(); + private final AtomicLong nextIngestJobId = new AtomicLong(0L); - static IngestJobScheduler getInstance() { + static IngestScheduler getInstance() { return instance; } - private IngestJobScheduler() { + private IngestScheduler() { } /** @@ -74,7 +70,7 @@ final class IngestJobScheduler { * @return A collection of ingest module start up errors, empty on success. * @throws InterruptedException */ - synchronized List startIngestJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { + List startIngestJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { long jobId = nextIngestJobId.incrementAndGet(); IngestJob job = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); ingestJobsById.put(jobId, job); @@ -99,10 +95,11 @@ final class IngestJobScheduler { } synchronized void addDataSourceToIngestJob(IngestJob job, Content dataSource) throws InterruptedException { + // Enqueue a data source ingest task for the data source. // If the thread executing this code is interrupted, it is because the // the number of ingest threads has been decreased while ingest jobs are // running. The calling thread will exit in an orderly fashion, but the - // task still needs to be enqueued rather than lost. + // task still needs to be enqueued rather than lost, hence the loop. DataSourceIngestTask task = new DataSourceIngestTask(job, dataSource); while (true) { try { @@ -115,24 +112,27 @@ final class IngestJobScheduler { } } + // Get the top level files of the data source. Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); - List firstLevelFiles = new ArrayList<>(); + List toptLevelFiles = new ArrayList<>(); if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { - // The data source is file. - firstLevelFiles.add((AbstractFile) dataSource); + // The data source is itself a file. + toptLevelFiles.add((AbstractFile) dataSource); } else { for (AbstractFile root : rootObjects) { List children; try { children = root.getChildren(); if (children.isEmpty()) { - //add the root itself, could be unalloc file, child of volume or image - firstLevelFiles.add(root); + // Add the root object itself, it could be an unallocated space + // file, or a child of a volume or an image. + toptLevelFiles.add(root); } else { - //root for fs root dir, schedule children dirs/files + // The root object is a file system root directory, get + // the files within it. for (Content child : children) { if (child instanceof AbstractFile) { - firstLevelFiles.add((AbstractFile) child); + toptLevelFiles.add((AbstractFile) child); } } } @@ -141,26 +141,30 @@ final class IngestJobScheduler { } } } - for (AbstractFile firstLevelFile : firstLevelFiles) { + + // Enqueue file ingest tasks for the top level files. + for (AbstractFile firstLevelFile : toptLevelFiles) { FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); if (shouldEnqueueFileTask(fileTask)) { rootDirectoryTasks.add(fileTask); - fileTask.getIngestJob().notifyTaskAdded(); } } - // Reshuffle/update the dir and file level queues if needed - updateFileTaskQueues(); + updateFileTaskQueues(null); } - synchronized void addFileToIngestJob(IngestJob job, AbstractFile file) { // RJCTODO: Just one at a time? + void addFileToIngestJob(IngestJob job, AbstractFile file) { FileIngestTask task = new FileIngestTask(job, file); if (shouldEnqueueFileTask(task)) { addTaskToFileQueue(task); - } + } } - private synchronized void updateFileTaskQueues() throws InterruptedException { + private synchronized void updateFileTaskQueues(FileIngestTask taskInProgress) throws InterruptedException { + if (taskInProgress != null) { + tasksInProgress.add(taskInProgress); + } + // we loop because we could have a directory that has all files // that do not get enqueued while (true) { @@ -194,7 +198,6 @@ final class IngestJobScheduler { FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); if (childFile.hasChildren()) { directoryTasks.add(childTask); - childTask.getIngestJob().notifyTaskAdded(); } else if (shouldEnqueueFileTask(childTask)) { addTaskToFileQueue(childTask); } @@ -205,7 +208,7 @@ final class IngestJobScheduler { } } } - + private void addTaskToFileQueue(FileIngestTask task) { // If the thread executing this code is interrupted, it is because the // the number of ingest threads has been decreased while ingest jobs are @@ -222,14 +225,7 @@ final class IngestJobScheduler { } } } - - /** - * Check if the file is a special file that we should skip - * - * @param processTask a task whose file to check if should be queued of - * skipped - * @return true if should be enqueued, false otherwise - */ + private static boolean shouldEnqueueFileTask(final FileIngestTask processTask) { final AbstractFile aFile = processTask.getFile(); //if it's unalloc file, skip if so scheduled @@ -282,68 +278,52 @@ final class IngestJobScheduler { } IngestTaskQueue getDataSourceIngestTaskQueue() { - return dataSourceIngestTaskQueue; + return dataSourceTaskDispenser; } IngestTaskQueue getFileIngestTaskQueue() { - return fileIngestTaskQueue; + return fileTaskDispenser; } - /** - * Finds top level objects such as file system root directories, layout - * files and virtual directories. - */ - private static class GetRootDirectoryVisitor extends GetFilesContentVisitor { - - @Override - public Collection visit(VirtualDirectory ld) { - //case when we hit a layout directoryor local file container, not under a real FS - //or when root virt dir is scheduled - Collection ret = new ArrayList<>(); - ret.add(ld); - return ret; + void ingestTaskIsCompleted(IngestTask completedTask) { + if (ingestJobIsCompleted(completedTask)) { + IngestJob job = completedTask.getIngestJob(); + job.shutDown(); + ingestJobsById.remove(job.getId()); + IngestManager.getInstance().fireIngestJobCompleted(job.getId()); } + } - @Override - public Collection visit(LayoutFile lf) { - //case when we hit a layout file, not under a real FS - Collection ret = new ArrayList<>(); - ret.add(lf); - return ret; + private synchronized boolean ingestJobIsCompleted(IngestTask completedTask) { + tasksInProgress.remove(completedTask); + IngestJob job = completedTask.getIngestJob(); + long jobId = job.getId(); + for (IngestTask task : tasksInProgress) { + if (task.getIngestJob().getId() == jobId) { + return false; + } } - - @Override - public Collection visit(Directory drctr) { - //we hit a real directory, a child of real FS - Collection ret = new ArrayList<>(); - ret.add(drctr); - return ret; + for (FileIngestTask task : fileTasks) { + if (task.getIngestJob().getId() == jobId) { + return false; + } } - - @Override - public Collection visit(FileSystem fs) { - return getAllFromChildren(fs); + for (FileIngestTask task : directoryTasks) { + if (task.getIngestJob().getId() == jobId) { + return false; + } } - - @Override - public Collection visit(File file) { - //can have derived files - return getAllFromChildren(file); + for (FileIngestTask task : rootDirectoryTasks) { + if (task.getIngestJob().getId() == jobId) { + return false; + } } - - @Override - public Collection visit(DerivedFile derivedFile) { - //can have derived files - //TODO test this and overall scheduler with derived files - return getAllFromChildren(derivedFile); - } - - @Override - public Collection visit(LocalFile localFile) { - //can have local files - //TODO test this and overall scheduler with local files - return getAllFromChildren(localFile); + for (DataSourceIngestTask task : dataSourceTasks) { + if (task.getIngestJob().getId() == jobId) { + return false; + } } + return true; } private static class RootDirectoryTaskComparator implements Comparator { @@ -455,7 +435,7 @@ final class IngestJobScheduler { @Override public IngestTask getNextTask() throws InterruptedException { FileIngestTask task = fileTasks.take(); - updateFileTaskQueues(); + updateFileTaskQueues(task); return task; } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index 1133be1210..414128fd5f 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -18,6 +18,17 @@ */ package org.sleuthkit.autopsy.ingest; -interface IngestTask { - void execute() throws InterruptedException; +abstract class IngestTask { + + private final IngestJob job; + + IngestTask(IngestJob job) { + this.job = job; + } + + IngestJob getIngestJob() { + return job; + } + + abstract void execute() throws InterruptedException; }