mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-12 16:06:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Safekeep ingest framework improvements

Browse Source
This commit is contained in:
Richard Cordovano 2014-04-29 17:49:12 -04:00
parent 45ac9090a5
commit ae1793bfd3
15 changed files with 1203 additions and 1196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties
View File

@ -52,12 +52,6 @@ Ensure the Case drive has at least 1GB free space and restart ingest.
IngestJobConfigurationPanel.advancedButton.text=Advanced IngestJobConfigurationPanel.advancedButton.text=Advanced
IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\: IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\:
IngestScheduler.FileSched.toString.curFiles.text=\
CurFiles, size\:
IngestScheduler.FileSched.toString.curDirs.text=\
CurDirs(stack), size\:
IngestScheduler.FileSched.toString.rootDirs.text=\
RootDirs(sorted), size\:
IngestManager.StartIngestJobsTask.run.displayName=Queueing ingest tasks IngestManager.StartIngestJobsTask.run.displayName=Queueing ingest tasks
IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...) IngestManager.StartIngestJobsTask.run.cancelling={0} (Cancelling...)
IngestManager.StartIngestJobsTask.run.catchException.msg=An error occurred while starting ingest. Results may only be partial IngestManager.StartIngestJobsTask.run.catchException.msg=An error occurred while starting ingest. Results may only be partial

130
Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties
View File

@ -1,74 +1,68 @@
CTL_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8 CTL_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8
HINT_IngestMessageTopComponent=\u30E1\u30C3\u30BB\u30FC\u30B8\u30A6\u30A3\u30F3\u30C9\u30A6 HINT_IngestMessageTopComponent=\u30e1\u30c3\u30bb\u30fc\u30b8\u30a6\u30a3\u30f3\u30c9\u30a6
IngestDialog.closeButton.title=\u9589\u3058\u308B IngestDialog.closeButton.title=\u9589\u3058\u308b
IngestDialog.startButton.title=\u30B9\u30BF\u30FC\u30C8 IngestDialog.startButton.title=\u30b9\u30bf\u30fc\u30c8
IngestDialog.title.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB IngestDialog.title.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb
IngestJob.progress.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 IngestJob.progress.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09
IngestJob.progress.dataSourceIngest.displayName={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 IngestJob.progress.dataSourceIngest.displayName={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
IngestJob.progress.fileIngest.displayName={0}\u306E\u30D5\u30A1\u30A4\u30EB\u3092\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8 IngestJob.progress.fileIngest.displayName={0}\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
IngestJobConfigurationPanel.advancedButton.actionCommand=\u30A2\u30C9\u30D0\u30F3\u30B9 IngestJobConfigurationPanel.advancedButton.actionCommand=\u30a2\u30c9\u30d0\u30f3\u30b9
IngestJobConfigurationPanel.advancedButton.text=\u30A2\u30C9\u30D0\u30F3\u30B9 IngestJobConfigurationPanel.advancedButton.text=\u30a2\u30c9\u30d0\u30f3\u30b9
IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u306E\u51E6\u7406 IngestJobConfigurationPanel.processUnallocCheckbox.text=\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u306e\u51e6\u7406
IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524A\u9664\u3055\u308C\u305F\u30D5\u30A1\u30A4\u30EB\u7B49\u306E\u672A\u5272\u308A\u5F53\u3066\u9818\u57DF\u3092\u51E6\u7406\u3002\u3088\u308A\u5B8C\u5168\u306A\u7D50\u679C\u304C\u51FA\u307E\u3059\u304C\u3001\u5927\u304D\u3044\u30A4\u30E1\u30FC\u30B8\u3067\u306F\u51E6\u7406\u6642\u9593\u304C\u9577\u304F\u306A\u308B\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=\u524a\u9664\u3055\u308c\u305f\u30d5\u30a1\u30a4\u30eb\u7b49\u306e\u672a\u5272\u308a\u5f53\u3066\u9818\u57df\u3092\u51e6\u7406\u3002\u3088\u308a\u5b8c\u5168\u306a\u7d50\u679c\u304c\u51fa\u307e\u3059\u304c\u3001\u5927\u304d\u3044\u30a4\u30e1\u30fc\u30b8\u3067\u306f\u51e6\u7406\u6642\u9593\u304c\u9577\u304f\u306a\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestManager.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC IngestManager.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc
IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestManager.moduleErr.errListenToUpdates.msg=Ingest Manager\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestManager.StartIngestJobsTask.run.cancelling={0}\uFF08\u30AD\u30E3\u30F3\u30BB\u30EB\u4E2D\u2026\uFF09 IngestManager.StartIngestJobsTask.run.cancelling={0}\uff08\u30ad\u30e3\u30f3\u30bb\u30eb\u4e2d\u2026\uff09
IngestManager.StartIngestJobsTask.run.catchException.msg=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u306E\u958B\u59CB\u4E2D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F\u3002\u7D50\u679C\u304C\u4E00\u90E8\u306E\u3082\u306E IngestManager.StartIngestJobsTask.run.catchException.msg=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u306e\u958b\u59cb\u4e2d\u306b\u30a8\u30e9\u30fc\u304c\u767a\u751f\u3057\u307e\u3057\u305f\u3002\u7d50\u679c\u304c\u4e00\u90e8\u306e\u3082\u306e
IngestManager.StartIngestJobsTask.run.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF\u3092\u30AD\u30E5\u30FC\u30A4\u30F3\u30B0 IngestManager.StartIngestJobsTask.run.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af\u3092\u30ad\u30e5\u30fc\u30a4\u30f3\u30b0
IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30BD\u30FC\u30B9\u3001\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u3001\u8A73\u7D30\u304A\u3088\u3073\u30C7\u30FC\u30BF\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.srcSubjDetailsDataNotNull.msg=\u30bd\u30fc\u30b9\u3001\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u3001\u8a73\u7d30\u304a\u3088\u3073\u30c7\u30fc\u30bf\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.exception.srcSubjNotNull.msg=\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.srcSubjNotNull.msg=\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.exception.typeSrcSubjNotNull.msg=\u30E1\u30C3\u30BB\u30FC\u30B8\u30BF\u30A4\u30D7\u3001\u30BD\u30FC\u30B9\u304A\u3088\u3073\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\u306F\u30CC\u30EB\u3067\u3042\u3063\u3066\u306F\u3044\u3051\u307E\u305B\u3093 IngestMessage.exception.typeSrcSubjNotNull.msg=\u30e1\u30c3\u30bb\u30fc\u30b8\u30bf\u30a4\u30d7\u3001\u30bd\u30fc\u30b9\u304a\u3088\u3073\u30b5\u30d6\u30b8\u30a7\u30af\u30c8\u306f\u30cc\u30eb\u3067\u3042\u3063\u3066\u306f\u3044\u3051\u307e\u305b\u3093
IngestMessage.toString.data.text=\ \u30C7\u30FC\u30BF\uFF1A{0} IngestMessage.toString.data.text=\ \u30c7\u30fc\u30bf\uff1a{0}
IngestMessage.toString.date.text=\ \u65E5\u4ED8\uFF1A{0} IngestMessage.toString.date.text=\ \u65e5\u4ed8\uff1a{0}
IngestMessage.toString.details.text=\ \u8A73\u7D30\uFF1A{0} IngestMessage.toString.details.text=\ \u8a73\u7d30\uff1a{0}
IngestMessage.toString.subject.text=\ \u30B5\u30D6\u30B8\u30A7\u30AF\u30C8\uFF1A{0} IngestMessage.toString.subject.text=\ \u30b5\u30d6\u30b8\u30a7\u30af\u30c8\uff1a{0}
IngestMessage.toString.type.text=\u30BF\u30A4\u30D7\uFF1A{0} IngestMessage.toString.type.text=\u30bf\u30a4\u30d7\uff1a{0}
IngestMessageDetailsPanel.copyMenuItem.text=\u30B3\u30D4\u30FC IngestMessageDetailsPanel.copyMenuItem.text=\u30b3\u30d4\u30fc
IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30C6\u30AD\u30B9\u30C8\uFF0Fhtml IngestMessageDetailsPanel.messageDetailsPane.contentType=\u30c6\u30ad\u30b9\u30c8\uff0fhtml
IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629E IngestMessageDetailsPanel.selectAllMenuItem.text=\u3059\u3079\u3066\u9078\u629e
IngestMessageDetailsPanel.viewArtifactButton.text=\u7D50\u679C\u3078\u79FB\u52D5 IngestMessageDetailsPanel.viewArtifactButton.text=\u7d50\u679c\u3078\u79fb\u52d5
IngestMessageDetailsPanel.viewContentButton.text=\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u3078\u79FB\u52D5 IngestMessageDetailsPanel.viewContentButton.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3078\u79fb\u52d5
IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30D6\u30FC\u30EB\u5024\u3067\u306F\u306A\u3044\u3082\u306E\u306BBooleanRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F IngestMessagePanel.BooleanRenderer.exception.nonBoolVal.msg=\u30d6\u30fc\u30eb\u5024\u3067\u306f\u306a\u3044\u3082\u306e\u306bBooleanRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f
IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65E5\u4ED8\u3067\u306F\u306A\u3044\u3082\u306E\u306BDateRenderer\u3092\u4F7F\u7528\u3057\u3088\u3046\u3068\u3057\u307E\u3057\u305F\u3002 IngestMessagePanel.DateRenderer.exception.nonDateVal.text=\u65e5\u4ed8\u3067\u306f\u306a\u3044\u3082\u306e\u306bDateRenderer\u3092\u4f7f\u7528\u3057\u3088\u3046\u3068\u3057\u307e\u3057\u305f\u3002
IngestMessagePanel.moduleErr=\u30E2\u30B8\u30E5\u30FC\u30EB\u30A8\u30E9\u30FC IngestMessagePanel.moduleErr=\u30e2\u30b8\u30e5\u30fc\u30eb\u30a8\u30e9\u30fc
IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30A2\u30C3\u30D7\u30C7\u30FC\u30C8\u3092\u78BA\u8A8D\u4E2D\u306B\u30E2\u30B8\u30E5\u30FC\u30EB\u304C\u30A8\u30E9\u30FC\u3092\u8D77\u3053\u3057\u307E\u3057\u305F\u3002\u3069\u306E\u30E2\u30B8\u30E5\u30FC\u30EB\u304B\u30ED\u30B0\u3067\u78BA\u8A8D\u3057\u3066\u4E0B\u3055\u3044\u3002\u4E00\u90E8\u306E\u30C7\u30FC\u30BF\u304C\u4E0D\u5B8C\u5168\u304B\u3082\u3057\u308C\u307E\u305B\u3093\u3002 IngestMessagePanel.moduleErr.errListenUpdates.text=IngestMessagePanel\u30a2\u30c3\u30d7\u30c7\u30fc\u30c8\u3092\u78ba\u8a8d\u4e2d\u306b\u30e2\u30b8\u30e5\u30fc\u30eb\u304c\u30a8\u30e9\u30fc\u3092\u8d77\u3053\u3057\u307e\u3057\u305f\u3002\u3069\u306e\u30e2\u30b8\u30e5\u30fc\u30eb\u304b\u30ed\u30b0\u3067\u78ba\u8a8d\u3057\u3066\u4e0b\u3055\u3044\u3002\u4e00\u90e8\u306e\u30c7\u30fc\u30bf\u304c\u4e0d\u5b8c\u5168\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002
IngestMessagePanel.MsgTableMod.colNames.module=\u30E2\u30B8\u30E5\u30FC\u30EB IngestMessagePanel.MsgTableMod.colNames.module=\u30e2\u30b8\u30e5\u30fc\u30eb
IngestMessagePanel.MsgTableMod.colNames.new=\u65B0\u898F\uFF1F IngestMessagePanel.MsgTableMod.colNames.new=\u65b0\u898f\uff1f
IngestMessagePanel.MsgTableMod.colNames.num=\u756A\u53F7 IngestMessagePanel.MsgTableMod.colNames.num=\u756a\u53f7
IngestMessagePanel.MsgTableMod.colNames.subject=\u30B5\u30D6\u30B8\u30A7\u30AF\u30C8 IngestMessagePanel.MsgTableMod.colNames.subject=\u30b5\u30d6\u30b8\u30a7\u30af\u30c8
IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30BF\u30A4\u30E0\u30B9\u30BF\u30F3\u30D7 IngestMessagePanel.MsgTableMod.colNames.timestamp=\u30bf\u30a4\u30e0\u30b9\u30bf\u30f3\u30d7
IngestMessagePanel.sortByComboBox.model.priority=\u512A\u5148\u5EA6 IngestMessagePanel.sortByComboBox.model.priority=\u512a\u5148\u5ea6
IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593 IngestMessagePanel.sortByComboBox.model.time=\u6642\u9593
IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uFF08\u6642\u7CFB\u5217\uFF09\u307E\u305F\u306F\u30E1\u30C3\u30BB\u30FC\u30B8\u306E\u512A\u5148\u5EA6\u3067\u30BD\u30FC\u30C8 IngestMessagePanel.sortByComboBox.toolTipText=\u6642\u9593\u9806\uff08\u6642\u7cfb\u5217\uff09\u307e\u305f\u306f\u30e1\u30c3\u30bb\u30fc\u30b8\u306e\u512a\u5148\u5ea6\u3067\u30bd\u30fc\u30c8
IngestMessagePanel.sortByLabel.text=\u4E0B\u8A18\u3067\u30BD\u30FC\u30C8\uFF1A IngestMessagePanel.sortByLabel.text=\u4e0b\u8a18\u3067\u30bd\u30fc\u30c8\uff1a
IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8A08\uFF1A IngestMessagePanel.totalMessagesNameLabel.text=\u5408\u8a08\uff1a
IngestMessagePanel.totalMessagesNameVal.text=- IngestMessagePanel.totalMessagesNameVal.text=-
IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30E6\u30CB\u30FC\u30AF\uFF1A IngestMessagePanel.totalUniqueMessagesNameLabel.text=\u30e6\u30cb\u30fc\u30af\uff1a
IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestMessagePanel.totalUniqueMessagesNameVal.text=-
IngestMessagesToolbar.customizeButton.toolTipText=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E1\u30C3\u30BB\u30FC\u30B8 IngestMessagesToolbar.customizeButton.toolTipText=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e1\u30c3\u30bb\u30fc\u30b8
IngestMessageTopComponent.displayName=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 IngestMessageTopComponent.displayName=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9
IngestMessageTopComponent.displayReport.option.GenRpt=\u30EC\u30DD\u30FC\u30C8\u751F\u6210 IngestMessageTopComponent.displayReport.option.GenRpt=\u30ec\u30dd\u30fc\u30c8\u751f\u6210
IngestMessageTopComponent.displayReport.option.OK=OK IngestMessageTopComponent.displayReport.option.OK=OK
IngestMessageTopComponent.initComponents.name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30A4\u30F3\u30DC\u30C3\u30AF\u30B9 IngestMessageTopComponent.initComponents.name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30a4\u30f3\u30dc\u30c3\u30af\u30b9
IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30EC\u30DD\u30FC\u30C8 IngestMessageTopComponent.msgDlg.ingestRpt.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30ec\u30dd\u30fc\u30c8
IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30C7\u30A3\u30B9\u30AF{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3\u306E\u305F\u3081\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u4E2D\u6B62\u3057\u307E\u3059\u3002\ IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3\u306e\u305f\u3081\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u4e2d\u6b62\u3057\u307e\u3059\u3002\
\u30B1\u30FC\u30B9\u30C9\u30E9\u30A4\u30D6\u306B\u6700\u4F4E1GB\u306E\u7A7A\u304D\u9818\u57DF\u304C\u3042\u308B\u306E\u3092\u78BA\u8A8D\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u518D\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 \u30b1\u30fc\u30b9\u30c9\u30e9\u30a4\u30d6\u306b\u6700\u4f4e1GB\u306e\u7a7a\u304d\u9818\u57df\u304c\u3042\u308b\u306e\u3092\u78ba\u8a8d\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u518d\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002
IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u304C\u4E2D\u6B62\u3055\u308C\u307E\u3057\u305F\u30FC{0}\u306E\u30C7\u30A3\u30B9\u30AF\u9818\u57DF\u4E0D\u8DB3 IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30B5\u30A4\u30BA\uFF1A IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30b5\u30a4\u30ba\uff1a
IngestScheduler.FileSched.toString.curDirs.text=\ OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
CurDirs(stack), \u30B5\u30A4\u30BA\uFF1A IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
IngestScheduler.FileSched.toString.curFiles.text=\ IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
CurFiles, \u30B5\u30A4\u30BA\uFF1A IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
IngestScheduler.FileSched.toString.rootDirs.text=\ IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
RootDirs(sorted), \u30B5\u30A4\u30BA\uFF1A IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30a8\u30e9\u30fc\uff1a\
OpenIDE-Module-Name=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8
IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306E\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30BF\u30B9\u30AF
IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30A8\u30E9\u30FC\uFF1A\
\ \
{0} {0}
IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uFF11\u3064\u307E\u305F\u306F\u8907\u6570\u306E\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u30B9\u30BF\u30FC\u30C8\u3067\u304D\u307E\u305B\u3093\u3067\u3057\u305F\u3002\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30B8\u30E7\u30D6\u306F\u30AD\u30E3\u30F3\u30BB\u30EB\u3055\u308C\u307E\u3057\u305F\u3002 IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=\uff11\u3064\u307e\u305f\u306f\u8907\u6570\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u30b9\u30bf\u30fc\u30c8\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\u3002\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30b8\u30e7\u30d6\u306f\u30ad\u30e3\u30f3\u30bb\u30eb\u3055\u308c\u307e\u3057\u305f\u3002
IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305F\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u7121\u52B9\u5316\u3059\u308B\u304B\u30A8\u30E9\u30FC\u3092\u89E3\u6C7A\u3057\u3001\u305D\u306E\u5F8C\u30C7\u30FC\u30BF\u30BD\u30FC\u30B9\u3092\u53F3\u30AF\u30EA\u30C3\u30AF\u3057\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u30E2\u30B8\u30E5\u30FC\u30EB\u3092\u5B9F\u884C\u3092\u9078\u629E\u3057\u3066\u3001\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u3092\u30EA\u30B9\u30BF\u30FC\u30C8\u3057\u3066\u4E0B\u3055\u3044\u3002 IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=\u5931\u6557\u3057\u305f\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u7121\u52b9\u5316\u3059\u308b\u304b\u30a8\u30e9\u30fc\u3092\u89e3\u6c7a\u3057\u3001\u305d\u306e\u5f8c\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u3092\u53f3\u30af\u30ea\u30c3\u30af\u3057\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u5b9f\u884c\u3092\u9078\u629e\u3057\u3066\u3001\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3092\u30ea\u30b9\u30bf\u30fc\u30c8\u3057\u3066\u4e0b\u3055\u3044\u3002
IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30A4\u30F3\u30B8\u30A7\u30B9\u30C8\u5931\u6557 IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u5931\u6557

4
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java
View File

@ -34,8 +34,8 @@ final class DataSourceIngestPipeline {
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>(); private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
DataSourceIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) { DataSourceIngestPipeline(IngestJob job, List<IngestModuleTemplate> moduleTemplates) {
this.job = task; this.job = job;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
} }

44
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java Executable file
View File

@ -0,0 +1,44 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import org.sleuthkit.datamodel.Content;
final class DataSourceIngestTask {
private final IngestJob ingestJob;
private final Content dataSource;
DataSourceIngestTask(IngestJob ingestJob, Content dataSource) {
this.ingestJob = ingestJob;
this.dataSource = dataSource;
}
IngestJob getIngestJob() {
return ingestJob;
}
Content getDataSource() {
return dataSource;
}
void execute() {
ingestJob.process();
}
}

57
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java Executable file
View File

@ -0,0 +1,57 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.concurrent.LinkedBlockingQueue;
final class DataSourceIngestTaskScheduler {
private static DataSourceIngestTaskScheduler instance = new DataSourceIngestTaskScheduler();
private final LinkedBlockingQueue<DataSourceIngestTask> tasks = new LinkedBlockingQueue<>();
static DataSourceIngestTaskScheduler getInstance() {
return instance;
}
private DataSourceIngestTaskScheduler() {
}
synchronized void addTask(DataSourceIngestTask task) throws InterruptedException {
task.getIngestJob().notifyTaskPending();
try {
tasks.put(task);
}
catch (InterruptedException ex) {
// RJCTOD: Need a safety notification to undo above
}
}
DataSourceIngestTask getNextTask() throws InterruptedException {
return tasks.take();
}
boolean hasTasksForIngestJob(long jobId) {
for (DataSourceIngestTask task : tasks) {
if (task.getIngestJobId() == jobId) {
return true;
}
}
return false;
}
}

93
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java Executable file
View File

@ -0,0 +1,93 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.Objects;
import java.util.logging.Level;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
final class FileIngestTask {
final AbstractFile file;
private final IngestJob ingestJob;
FileIngestTask(AbstractFile file, IngestJob task) {
this.file = file;
this.ingestJob = task;
}
public IngestJob getIngestJob() {
return ingestJob;
}
public AbstractFile getFile() {
return file;
}
void execute(long threadId) {
ingestJob.process(file);
}
@Override
public String toString() { //RJCTODO: May not keep this
try {
return "ProcessTask{" + "file=" + file.getId() + ": " + file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}';
} catch (TskCoreException ex) {
// RJCTODO
// FileIngestTaskScheduler.logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS
}
return "ProcessTask{" + "file=" + file.getId() + ": " + file.getName() + '}';
}
/**
* two process tasks are equal when the file/dir and modules are the
* same this enables are not to queue up the same file/dir, modules
* tuples into the root dir set
*
* @param obj
* @return
*/
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final FileIngestTask other = (FileIngestTask) obj;
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
return false;
}
IngestJob thisTask = this.getIngestJob();
IngestJob otherTask = other.getIngestJob();
if (thisTask != otherTask && (thisTask == null || !thisTask.equals(otherTask))) {
return false;
}
return true;
}
@Override
public int hashCode() {
int hash = 5;
hash = 47 * hash + Objects.hashCode(this.file);
hash = 47 * hash + Objects.hashCode(this.ingestJob);
return hash;
}
}

418
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java Executable file
View File

@ -0,0 +1,418 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.TreeSet;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.VirtualDirectory;
final class FileIngestTaskScheduler {
private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName());
private static FileIngestTaskScheduler instance;
private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator());
private final List<FileIngestTask> directoryTasks = new ArrayList<>();
private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>(); // Unlimited capacity
private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
static synchronized FileIngestTaskScheduler getInstance() {
if (instance == null) {
instance = new FileIngestTaskScheduler();
}
return instance;
}
private FileIngestTaskScheduler() {
}
synchronized void addTasks(IngestJob dataSourceTask, Content dataSource) {
Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
List<AbstractFile> firstLevelFiles = new ArrayList<>();
if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
// The data source is file.
firstLevelFiles.add((AbstractFile) dataSource);
} else {
for (AbstractFile root : rootObjects) {
List<Content> children;
try {
children = root.getChildren();
if (children.isEmpty()) {
//add the root itself, could be unalloc file, child of volume or image
firstLevelFiles.add(root);
} else {
//root for fs root dir, schedule children dirs/files
for (Content child : children) {
if (child instanceof AbstractFile) {
firstLevelFiles.add((AbstractFile) child);
}
}
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS
}
}
}
for (AbstractFile firstLevelFile : firstLevelFiles) {
FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask);
if (shouldEnqueueTask(fileTask)) {
rootDirectoryTasks.add(fileTask);
}
}
// Reshuffle/update the dir and file level queues if needed
updateQueues();
}
synchronized void addTask(IngestJob ingestJob, AbstractFile file) {
try {
FileIngestTask fileTask = new FileIngestTask(file, ingestJob);
if (shouldEnqueueTask(fileTask)) {
fileTask.getIngestJob().notifyTaskPending();
fileTasks.put(fileTask); // Queue has unlimited capacity, does not block.
}
} catch (InterruptedException ex) {
// RJCTODO: Perhaps this is the convenience method?
// RJCTODO: Need undo
}
}
FileIngestTask getNextTask() throws InterruptedException {
FileIngestTask task = fileTasks.take();
updateQueues();
return task;
}
synchronized boolean hasTasksForJob(long ingestJobId) {
for (FileIngestTask task : rootDirectoryTasks) {
if (task.getIngestJob().getJobId() == ingestJobId) {
return true;
}
}
for (FileIngestTask task : directoryTasks) {
if (task.getIngestJob().getJobId() == ingestJobId) {
return true;
}
}
for (FileIngestTask task : fileTasks) {
if (task.getIngestJob().getJobId() == ingestJobId) {
return true;
}
}
return false;
}
private void updateQueues() {
// we loop because we could have a directory that has all files
// that do not get enqueued
while (true) {
// There are files in the queue, we're done
if (fileTasks.isEmpty() == false) {
return;
}
// fill in the directory queue if it is empty.
if (this.directoryTasks.isEmpty()) {
// bail out if root is also empty -- we are done
if (rootDirectoryTasks.isEmpty()) {
return;
}
FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst();
directoryTasks.add(rootTask);
}
//pop and push AbstractFile directory children if any
//add the popped and its leaf children onto cur file list
FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1);
final AbstractFile parentFile = parentTask.file;
// add itself to the file list
if (shouldEnqueueTask(parentTask)) {
// RJCTODO
try {
parentTask.getIngestJob().notifyTaskPending();
fileTasks.put(parentTask);
} catch (InterruptedException ex) {
// RJCTODO: Maybe make a convenience method
// RJCTODO: Need undo
}
}
// add its children to the file and directory lists
try {
List<Content> children = parentFile.getChildren();
for (Content c : children) {
if (c instanceof AbstractFile) {
AbstractFile childFile = (AbstractFile) c;
FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getIngestJob());
if (childFile.hasChildren()) {
this.directoryTasks.add(childTask);
} else if (shouldEnqueueTask(childTask)) {
// RJCTODO
try {
childTask.getIngestJob().notifyTaskPending();
fileTasks.put(childTask);
} catch (InterruptedException ex) {
// RJCTODO: Maybe make a convenience method
// RJCTODO: Need undo
}
}
}
}
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Could not get children of file and update file queues: " + parentFile.getName(), ex);
}
}
}
synchronized void emptyQueues() { // RJCTODO: Perhaps clear all...
this.rootDirectoryTasks.clear();
this.directoryTasks.clear();
this.fileTasks.clear();
}
/**
* Check if the file is a special file that we should skip
*
* @param processTask a task whose file to check if should be queued of
* skipped
* @return true if should be enqueued, false otherwise
*/
private static boolean shouldEnqueueTask(final FileIngestTask processTask) {
final AbstractFile aFile = processTask.file;
//if it's unalloc file, skip if so scheduled
if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
return false;
}
String fileName = aFile.getName();
if (fileName.equals(".") || fileName.equals("..")) {
return false;
} else if (aFile instanceof org.sleuthkit.datamodel.File) {
final org.sleuthkit.datamodel.File f = (File) aFile;
//skip files in root dir, starting with $, containing : (not default attributes)
//with meta address < 32, i.e. some special large NTFS and FAT files
FileSystem fs = null;
try {
fs = f.getFileSystem();
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS
}
TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP;
if (fs != null) {
fsType = fs.getFsType();
}
if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) {
//not fat or ntfs, accept all files
return true;
}
boolean isInRootDir = false;
try {
isInRootDir = f.getParentDirectory().isRoot();
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS
}
if (isInRootDir && f.getMetaAddr() < 32) {
String name = f.getName();
if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) {
return false;
}
} else {
return true;
}
}
return true;
}
/**
* Visitor that gets a collection of top level objects to be scheduled, such
* as root directories (if there is FS) or LayoutFiles and virtual
* directories, also if there is no FS.
*/
static class GetRootDirectoryVisitor extends GetFilesContentVisitor {
@Override
public Collection<AbstractFile> visit(VirtualDirectory ld) {
//case when we hit a layout directoryor local file container, not under a real FS
//or when root virt dir is scheduled
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(ld);
return ret;
}
@Override
public Collection<AbstractFile> visit(LayoutFile lf) {
//case when we hit a layout file, not under a real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(lf);
return ret;
}
@Override
public Collection<AbstractFile> visit(Directory drctr) {
//we hit a real directory, a child of real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(drctr);
return ret;
}
@Override
public Collection<AbstractFile> visit(FileSystem fs) {
return getAllFromChildren(fs);
}
@Override
public Collection<AbstractFile> visit(File file) {
//can have derived files
return getAllFromChildren(file);
}
@Override
public Collection<AbstractFile> visit(DerivedFile derivedFile) {
//can have derived files
//TODO test this and overall scheduler with derived files
return getAllFromChildren(derivedFile);
}
@Override
public Collection<AbstractFile> visit(LocalFile localFile) {
//can have local files
//TODO test this and overall scheduler with local files
return getAllFromChildren(localFile);
}
}
/**
* Root directory sorter
*/
private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> {
@Override
public int compare(FileIngestTask q1, FileIngestTask q2) {
AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file);
AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file);
if (p1 == p2) {
return (int) (q2.file.getId() - q1.file.getId());
} else {
return p2.ordinal() - p1.ordinal();
}
}
/**
* Priority determination for sorted AbstractFile, used by
* RootDirComparator
*/
private static class AbstractFilePriority {
enum Priority {
LAST, LOW, MEDIUM, HIGH
}
static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>();
static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>();
static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>();
static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>();
/* prioritize root directory folders based on the assumption that we are
* looking for user content. Other types of investigations may want different
* priorities. */
static /* prioritize root directory folders based on the assumption that we are
* looking for user content. Other types of investigations may want different
* priorities. */ {
// these files have no structure, so they go last
//unalloc files are handled as virtual files in getPriority()
//LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
//LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE));
LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
// orphan files are often corrupt and windows does not typically have
// user content, so put them towards the bottom
LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
// all other files go into the medium category too
MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
// user content is top priority
HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE));
HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE));
}
/**
* Get the scheduling priority for a given file.
*
* @param abstractFile
* @return
*/
static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) {
if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
//quickly filter out unstructured content
//non-fs virtual files and dirs, such as representing unalloc space
return AbstractFilePriority.Priority.LAST;
}
//determine the fs files priority by name
final String path = abstractFile.getName();
if (path == null) {
return AbstractFilePriority.Priority.MEDIUM;
}
for (Pattern p : HIGH_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.HIGH;
}
}
for (Pattern p : MEDIUM_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.MEDIUM;
}
}
for (Pattern p : LOW_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.LOW;
}
}
for (Pattern p : LAST_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.LAST;
}
}
//default is medium
return AbstractFilePriority.Priority.MEDIUM;
}
}
}
}

99
Core/src/org/sleuthkit/autopsy/ingest/GetFilesCountVisitor.java Executable file
View File

@ -0,0 +1,99 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
* Get counts of ingestable files/dirs for the content input source.
*
* Note, also includes counts of all unalloc children files (for the fs, image,
* volume) even if ingest didn't ask for them
*/
final class GetFilesCountVisitor extends ContentVisitor.Default<Long> {
private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName());
@Override
public Long visit(FileSystem fs) {
//recursion stop here
//case of a real fs, query all files for it
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
StringBuilder queryB = new StringBuilder();
queryB.append("( (fs_obj_id = ").append(fs.getId()); //NON-NLS
//queryB.append(") OR (fs_obj_id = NULL) )");
queryB.append(") )");
queryB.append(" AND ( (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS
queryB.append(") OR (meta_type = ").append(TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS
queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS
queryB.append(") )");
//queryB.append( "AND (type = ");
//queryB.append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.getFileType());
//queryB.append(")");
try {
final String query = queryB.toString();
logger.log(Level.INFO, "Executing count files query: {0}", query); //NON-NLS
return sc.countFilesWhere(query);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); //NON-NLS
return 0L;
}
}
@Override
public Long visit(LayoutFile lf) {
//recursion stop here
//case of LayoutFile child of Image or Volume
return 1L;
}
private long getCountFromChildren(Content content) {
long count = 0;
try {
List<Content> children = content.getChildren();
if (children.size() > 0) {
for (Content child : children) {
count += child.accept(this);
}
} else {
count = 1;
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not get count of objects from children to get num of total files to be ingested", ex); //NON-NLS
}
return count;
}
@Override
protected Long defaultVisit(Content cntnt) {
//recurse assuming this is image/vs/volume
//recursion stops at fs or unalloc file
return getCountFromChildren(cntnt);
}
}

290
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -19,12 +19,16 @@
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory; import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable; import org.openide.util.Cancellable;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
/** /**
@ -33,31 +37,63 @@ import org.sleuthkit.datamodel.Content;
*/ */
final class IngestJob { final class IngestJob {
private final long id; private static final AtomicLong nextIngestJobId = new AtomicLong(0L);
private static final ConcurrentHashMap<Long, IngestJob> ingestJobs = new ConcurrentHashMap<>(); // Maps job ids to jobs.
private final long jobId;
private final Content dataSource; private final Content dataSource;
private final List<IngestModuleTemplate> ingestModuleTemplates; private final List<IngestModuleTemplate> ingestModuleTemplates;
private final boolean processUnallocatedSpace; private final boolean processUnallocatedSpace;
private final HashMap<Long, FileIngestPipeline> fileIngestPipelines = new HashMap<>(); private final LinkedBlockingQueue<DataSourceIngestPipeline> dataSourceIngestPipelines = new LinkedBlockingQueue<>();
private final HashMap<Long, DataSourceIngestPipeline> dataSourceIngestPipelines = new HashMap<>(); private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelines = new LinkedBlockingQueue<>();
private final IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); private final AtomicInteger tasksInProgress = new AtomicInteger(0);
private FileIngestPipeline initialFileIngestPipeline = null; private final AtomicLong processedFiles = new AtomicLong(0L);
private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; private ProgressHandle dataSourceTasksProgress;
private ProgressHandle dataSourceTaskProgress;
private ProgressHandle fileTasksProgress; private ProgressHandle fileTasksProgress;
int totalEnqueuedFiles = 0; private long filesToIngestEstimate = 0;
private int processedFiles = 0;
private volatile boolean cancelled; private volatile boolean cancelled;
IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) { static List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) { // RJCTODO: return errors
this.id = id; long jobId = nextIngestJobId.incrementAndGet();
IngestJob ingestJob = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace);
List<IngestModuleError> errors = ingestJob.start();
if (errors.isEmpty()) {
ingestJobs.put(jobId, ingestJob);
}
return errors;
}
static boolean jobsAreRunning() {
for (IngestJob job : ingestJobs.values()) {
if (!job.isCancelled()) {
return true;
}
}
return false;
}
static void addFileToIngestJob(long ingestJobId, AbstractFile file) { // RJCTODO: Move back to IngestManager
IngestJob job = ingestJobs.get(ingestJobId);
if (job != null) {
FileIngestTaskScheduler.getInstance().addTask(job, file);
}
}
static void cancelAllIngestJobs() {
for (IngestJob job : ingestJobs.values()) {
job.cancel();
}
}
private IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) {
this.jobId = id;
this.dataSource = dataSource; this.dataSource = dataSource;
this.ingestModuleTemplates = ingestModuleTemplates; this.ingestModuleTemplates = ingestModuleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace; this.processUnallocatedSpace = processUnallocatedSpace;
this.cancelled = false; this.cancelled = false;
} }
long getId() { long getJobId() {
return id; return jobId;
} }
Content getDataSource() { Content getDataSource() {
@ -68,34 +104,78 @@ final class IngestJob {
return processUnallocatedSpace; return processUnallocatedSpace;
} }
synchronized List<IngestModuleError> startUpIngestPipelines() { List<IngestModuleError> start() {
List<IngestModuleError> errors = startUpIngestPipelines();
if (errors.isEmpty()) {
DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, dataSource));
FileIngestTaskScheduler.getInstance().addTasks(this, dataSource);
startDataSourceIngestProgressBar(); startDataSourceIngestProgressBar();
startFileIngestProgressBar(); startFileIngestProgressBar();
return startUpInitialIngestPipelines(); }
return errors;
}
private List<IngestModuleError> startUpIngestPipelines() {
List<IngestModuleError> errors = new ArrayList<>();
int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
try {
dataSourceIngestPipelines.put(pipeline);
} catch (InterruptedException ex) {
// RJCTODO: log unexpected block and interrupt, or throw
}
if (errors.isEmpty()) {
// No need to accumulate presumably redundant erros.
break;
}
}
maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
FileIngestPipeline pipeline = new FileIngestPipeline(this, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
try {
fileIngestPipelines.put(pipeline);
} catch (InterruptedException ex) {
// RJCTODO: log unexpected block and interrupt, or throw
}
if (errors.isEmpty()) {
// No need to accumulate presumably redundant erros.
break;
}
}
return errors;
} }
private void startDataSourceIngestProgressBar() { private void startDataSourceIngestProgressBar() {
final String displayName = NbBundle final String displayName = NbBundle.getMessage(this.getClass(),
.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", this.dataSource.getName()); "IngestJob.progress.dataSourceIngest.displayName",
dataSourceTaskProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { dataSource.getName());
dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
if (dataSourceTaskProgress != null) { if (dataSourceTasksProgress != null) {
dataSourceTaskProgress.setDisplayName(NbBundle.getMessage(this.getClass(), dataSourceTasksProgress.setDisplayName(
NbBundle.getMessage(this.getClass(),
"IngestJob.progress.cancelling", "IngestJob.progress.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); IngestJob.this.cancel();
return true; return true;
} }
}); });
dataSourceTaskProgress.start(); dataSourceTasksProgress.start();
dataSourceTaskProgress.switchToIndeterminate(); dataSourceTasksProgress.switchToIndeterminate(); // RJCTODO: check out the logic in the pipleine class
} }
private void startFileIngestProgressBar() { private void startFileIngestProgressBar() {
final String displayName = NbBundle final String displayName = NbBundle.getMessage(this.getClass(),
.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); "IngestJob.progress.fileIngest.displayName",
dataSource.getName());
fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
@ -104,124 +184,90 @@ final class IngestJob {
NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); IngestJob.this.cancel();
return true; return true;
} }
}); });
filesToIngestEstimate = dataSource.accept(new GetFilesCountVisitor());
fileTasksProgress.start(); fileTasksProgress.start();
fileTasksProgress.switchToIndeterminate(); fileTasksProgress.switchToDeterminate((int) filesToIngestEstimate);
totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
} }
private List<IngestModuleError> startUpInitialIngestPipelines() { /**
// Create a per thread instance of each pipeline type right now to make * Called by the ingest task schedulers when an ingest task for this ingest
// (reasonably) sure that the ingest modules can be started. * job is added to the scheduler's task queue.
initialDataSourceIngestPipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); */
initialFileIngestPipeline = new FileIngestPipeline(this, ingestModuleTemplates); void notifyTaskScheduled() {
List<IngestModuleError> errors = new ArrayList<>(); // Increment the task counter when a task is scheduled so that there is
errors.addAll(initialDataSourceIngestPipeline.startUp()); // a persistent record of the task's existence even after it is removed
errors.addAll(initialFileIngestPipeline.startUp()); // from the scheduler by an ingest thread. The task counter is used by
return errors; // the job to determine when it is done.
tasksInProgress.incrementAndGet();
} }
synchronized DataSourceIngestPipeline getDataSourceIngestPipelineForThread(long threadId) { /**
DataSourceIngestPipeline pipeline; * Called by the ingest schedulers as an "undo" operation for
if (initialDataSourceIngestPipeline != null) { * notifyTaskScheduled().
pipeline = initialDataSourceIngestPipeline; */
initialDataSourceIngestPipeline = null; void notifyTaskCompleted() {
dataSourceIngestPipelines.put(threadId, pipeline); // Decrement the task counter when a task is discarded by a scheduler.
} else if (!dataSourceIngestPipelines.containsKey(threadId)) { // The task counter is used by the job to determine when it is done.
pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); tasksInProgress.decrementAndGet();
pipeline.startUp();
dataSourceIngestPipelines.put(threadId, pipeline);
} else {
pipeline = dataSourceIngestPipelines.get(threadId);
}
return pipeline;
} }
synchronized FileIngestPipeline getFileIngestPipelineForThread(long threadId) { void process() throws InterruptedException {
FileIngestPipeline pipeline; if (!isCancelled()) {
if (initialFileIngestPipeline != null) { try {
pipeline = initialFileIngestPipeline; DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take();
initialFileIngestPipeline = null; pipeline.process(); // RJCTODO: Pass data source through?
fileIngestPipelines.put(threadId, pipeline); dataSourceIngestPipelines.put(pipeline);
} else if (!fileIngestPipelines.containsKey(threadId)) { } catch (InterruptedException ex) {
pipeline = new FileIngestPipeline(this, ingestModuleTemplates); // RJCTODO:
pipeline.startUp();
fileIngestPipelines.put(threadId, pipeline);
} else {
pipeline = fileIngestPipelines.get(threadId);
} }
return pipeline; }
ifCompletedShutDown();
} }
synchronized List<IngestModuleError> releaseIngestPipelinesForThread(long threadId) { void process(AbstractFile file) {
List<IngestModuleError> errors = new ArrayList<>(); if (!isCancelled()) {
try {
DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId); FileIngestPipeline pipeline = fileIngestPipelines.take();
if (dataSourceIngestPipeline != null) { fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet());
errors.addAll(dataSourceIngestPipeline.shutDown(cancelled)); pipeline.process(file);
dataSourceIngestPipelines.remove(threadId); fileIngestPipelines.put(pipeline);
} catch (InterruptedException ex) {
// RJCTODO: Log block and interrupt
} }
if (initialDataSourceIngestPipeline == null && dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) { }
dataSourceTaskProgress.finish(); ifCompletedShutDown();
dataSourceTaskProgress = null;
} }
FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId); void ifCompletedShutDown() {
if (fileIngestPipeline != null) { if (tasksInProgress.decrementAndGet() == 0) {
errors.addAll(fileIngestPipeline.shutDown(cancelled)); while (!dataSourceIngestPipelines.isEmpty()) {
fileIngestPipelines.remove(threadId); DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll();
pipeline.shutDown(cancelled);
}
while (!fileIngestPipelines.isEmpty()) {
FileIngestPipeline pipeline = fileIngestPipelines.poll();
pipeline.shutDown(cancelled);
}
ingestJobs.remove(jobId);
IngestManager.getInstance().fireIngestJobCompleted(jobId);
} }
if (initialFileIngestPipeline == null && fileIngestPipelines.isEmpty() && fileTasksProgress != null) {
fileTasksProgress.finish();
fileTasksProgress = null;
} }
return errors; ProgressHandle getDataSourceTaskProgressBar() {
} return dataSourceTasksProgress; // RJCTODO: Should just pass the progress handle or the object to the pipeline
synchronized boolean areIngestPipelinesShutDown() {
return (initialDataSourceIngestPipeline == null
&& dataSourceIngestPipelines.isEmpty()
&& initialFileIngestPipeline == null
&& fileIngestPipelines.isEmpty());
}
synchronized ProgressHandle getDataSourceTaskProgressBar() {
return this.dataSourceTaskProgress;
}
synchronized void updateFileTasksProgressBar(String currentFileName) {
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
totalEnqueuedFiles = newTotalEnqueuedFiles + 1;
fileTasksProgress.switchToIndeterminate();
fileTasksProgress.switchToDeterminate(totalEnqueuedFiles);
}
if (processedFiles < totalEnqueuedFiles) {
++processedFiles;
}
fileTasksProgress.progress(currentFileName, processedFiles);
}
synchronized void cancel() {
if (initialDataSourceIngestPipeline != null) {
initialDataSourceIngestPipeline.shutDown(true);
initialDataSourceIngestPipeline = null;
}
if (initialFileIngestPipeline != null) {
initialFileIngestPipeline.shutDown(true);
initialFileIngestPipeline = null;
}
cancelled = true;
} }
boolean isCancelled() { boolean isCancelled() {
return cancelled; return cancelled;
} }
void cancel() {
cancelled = true;
fileTasksProgress.finish();
IngestManager.getInstance().fireIngestJobCancelled(jobId);
}
} }

4
Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java
View File

@ -39,7 +39,7 @@ public final class IngestJobContext {
* @return The ingest job identifier. * @return The ingest job identifier.
*/ */
public long getJobId() { public long getJobId() {
return this.ingestJob.getId(); return this.ingestJob.getJobId();
} }
/** /**
@ -60,7 +60,7 @@ public final class IngestJobContext {
*/ */
public void addFiles(List<AbstractFile> files) { public void addFiles(List<AbstractFile> files) {
for (AbstractFile file : files) { for (AbstractFile file : files) {
IngestManager.getInstance().addFileToIngestJob(ingestJob.getId(), file); IngestJob.addFileToIngestJob(ingestJob.getJobId(), file);
} }
} }
} }

474
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -20,9 +20,7 @@ package org.sleuthkit.autopsy.ingest;
import java.beans.PropertyChangeListener; import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport; import java.beans.PropertyChangeSupport;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
@ -36,72 +34,146 @@ import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable; import org.openide.util.Cancellable;
import org.openide.util.NbPreferences; import org.openide.util.NbPreferences;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
import java.util.prefs.Preferences; import java.util.prefs.Preferences;
import javax.swing.JOptionPane; import javax.swing.JOptionPane;
import javax.swing.SwingWorker;
import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask;
/** /**
* Manages the execution of ingest jobs. * Manages the execution of ingest jobs.
*/ */
public class IngestManager { public class IngestManager {
private static final int MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS = 1;
private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS private static final String NUMBER_OF_FILE_INGEST_THREADS_KEY = "NumberOfFileingestThreads"; //NON-NLS
private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1; private static final int MIN_NUMBER_OF_FILE_INGEST_THREADS = 1;
private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4; private static final int MAX_NUMBER_OF_FILE_INGEST_THREADS = 4;
private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2;
private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class);
private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class);
private static final IngestManager instance = new IngestManager(); private static final IngestManager instance = new IngestManager();
private final IngestScheduler scheduler = IngestScheduler.getInstance(); private final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class);
private final IngestMonitor ingestMonitor = new IngestMonitor(); private final IngestMonitor ingestMonitor = new IngestMonitor();
private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); private final ConcurrentHashMap<Long, Future<?>> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ExecutorService fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService fireEventTasksExecutor = Executors.newSingleThreadExecutor(); private final ConcurrentHashMap<Long, Future<?>> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ConcurrentHashMap<Long, IngestJob> ingestJobs = new ConcurrentHashMap<>(1, 0.9f, 4); // Maps job ids to jobs. private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS);
private final ConcurrentHashMap<Long, Future<?>> ingestTasks = new ConcurrentHashMap<>(); // Maps task ids to task cancellation handles. Guarded by this. private final ExecutorService fireIngestJobEventsThreadPool = Executors.newSingleThreadExecutor();
private final AtomicLong ingestJobId = new AtomicLong(0L); private final ConcurrentHashMap<Long, Future<?>> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final AtomicLong ingestTaskId = new AtomicLong(0L); private final AtomicLong nextThreadId = new AtomicLong(0L);
private volatile IngestMessageTopComponent ingestMessageBox; private volatile IngestMessageTopComponent ingestMessageBox;
/** /**
* Gets the IngestManager singleton, creating it if necessary. * Gets the ingest manager.
* *
* @returns The IngestManager singleton. * @returns A singleton IngestManager object.
*/ */
public static IngestManager getInstance() { public static IngestManager getInstance() {
return instance; return instance;
} }
/**
* Starts the ingest monitor and the data source ingest and file ingest
* threads.
*/
private IngestManager() { private IngestManager() {
startDataSourceIngestThread();
int numberOfFileIngestThreads = getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfFileIngestThreads; ++i) {
startFileIngestThread();
}
} }
/** /**
* Signals to the ingest manager that it can go find the top component for * Signals to the ingest manager that it can go about finding the top
* the ingest messages in box. Called by the custom installer for this * component for the ingest messages in box. Called by the custom installer
* package once the window system is initialized. * for this package once the window system is initialized.
*/ */
void initIngestMessageInbox() { void initIngestMessageInbox() {
if (this.ingestMessageBox == null) { if (ingestMessageBox == null) {
this.ingestMessageBox = IngestMessageTopComponent.findInstance(); ingestMessageBox = IngestMessageTopComponent.findInstance();
} }
} }
/**
* Gets the maximum number of data source ingest threads the ingest manager
* will use.
*/
public static int getMaxNumberOfDataSourceIngestThreads() {
return MAX_NUMBER_OF_DATA_SOURCE_INGEST_THREADS;
}
/**
* Gets the maximum number of file ingest threads the ingest manager will
* use.
*/
public static int getMaxNumberOfFileIngestThreads() {
return MAX_NUMBER_OF_FILE_INGEST_THREADS;
}
/**
* Gets the number of file ingest threads the ingest manager will use.
*/
public synchronized static int getNumberOfFileIngestThreads() { public synchronized static int getNumberOfFileIngestThreads() {
return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS);
} }
/**
* Changes the number of file ingest threads the ingest manager will use to
* no more than MAX_NUMBER_OF_FILE_INGEST_THREADS and no less than
* MIN_NUMBER_OF_FILE_INGEST_THREADS. Out of range requests are converted to
* requests for DEFAULT_NUMBER_OF_FILE_INGEST_THREADS.
*
* @param numberOfThreads The desired number of file ingest threads.
*/
public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) { public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) {
if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS if ((numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS) || (numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS)) {
|| numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) {
numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS;
} }
userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads);
if (instance.fileIngestThreads.size() != numberOfThreads) {
if (instance.fileIngestThreads.size() > numberOfThreads) {
Long[] threadIds = instance.fileIngestThreads.keySet().toArray(new Long[instance.fileIngestThreads.size()]);
int numberOfThreadsToCancel = instance.fileIngestThreads.size() - numberOfThreads;
for (int i = 0; i < numberOfThreadsToCancel; ++i) {
instance.cancelFileIngestThread(threadIds[i]);
}
} else if (instance.fileIngestThreads.size() < numberOfThreads) {
int numberOfThreadsToAdd = numberOfThreads - instance.fileIngestThreads.size();
for (int i = 0; i < numberOfThreadsToAdd; ++i) {
instance.startFileIngestThread();
}
}
}
}
/**
* Submits a DataSourceIngestThread Runnable to the data source ingest
* thread pool.
*/
private void startDataSourceIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread(threadId));
dataSourceIngestThreads.put(threadId, handle);
}
/**
* Submits a DataSourceIngestThread Runnable to the data source ingest
* thread pool.
*/
private void startFileIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = fileIngestThreadPool.submit(new FileIngestThread(threadId));
fileIngestThreads.put(threadId, handle);
}
/**
* Cancels a DataSourceIngestThread Runnable in the file ingest thread pool.
*/
private void cancelFileIngestThread(long threadId) {
Future<?> handle = fileIngestThreads.remove(threadId);
handle.cancel(true);
} }
synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) { synchronized void startIngestJobs(final List<Content> dataSources, final List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
@ -109,9 +181,9 @@ public class IngestManager {
ingestMessageBox.clearMessages(); ingestMessageBox.clearMessages();
} }
long taskId = ingestTaskId.incrementAndGet(); long taskId = nextThreadId.incrementAndGet();
Future<?> task = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); Future<?> task = startIngestJobsThreadPool.submit(new StartIngestJobsThread(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
ingestTasks.put(taskId, task); fileIngestThreads.put(taskId, task);
if (ingestMessageBox != null) { if (ingestMessageBox != null) {
ingestMessageBox.restoreMessages(); ingestMessageBox.restoreMessages();
@ -121,43 +193,42 @@ public class IngestManager {
/** /**
* Test if any ingest jobs are in progress. * Test if any ingest jobs are in progress.
* *
* @return True if any ingest jobs are in progress, false otherwise * @return True if any ingest jobs are in progress, false otherwise.
*/ */
public boolean isIngestRunning() { public boolean isIngestRunning() {
return (ingestJobs.isEmpty() == false); return IngestJob.jobsAreRunning();
} }
void addFileToIngestJob(long ingestJobId, AbstractFile file) { public void cancelAllIngestJobs() {
IngestJob job = ingestJobs.get(ingestJobId); cancelStartIngestJobsTasks();
if (job != null) { IngestJob.cancelAllIngestJobs();
scheduler.getFileIngestScheduler().queueFile(job, file);
}
} }
void cancelIngestJobs() { private void cancelStartIngestJobsTasks() {
new IngestCancellationWorker().execute(); for (Future<?> future : startIngestJobThreads.values()) {
future.cancel(true);
}
startIngestJobThreads.clear();
} }
/** /**
* Ingest events. * Ingest events.
*/ */
public enum IngestEvent { public enum IngestEvent { // RJCTODO: Update comments if time permits
/** /**
* Property change event fired when an ingest job is started. The ingest * Property change event fired when an ingest job is started. The old
* job id is in old value field of the PropertyChangeEvent object. * and new values of the PropertyChangeEvent object are set to null.
*/ */
INGEST_JOB_STARTED, INGEST_JOB_STARTED,
/** /**
* Property change event fired when an ingest job is completed. The * Property change event fired when an ingest job is completed. The old
* ingest job id is in old value field of the PropertyChangeEvent * and new values of the PropertyChangeEvent object are set to null.
* object.
*/ */
INGEST_JOB_COMPLETED, INGEST_JOB_COMPLETED,
/** /**
* Property change event fired when an ingest job is canceled. The * Property change event fired when an ingest job is canceled. The old
* ingest job id is in old value field of the PropertyChangeEvent * and new values of the PropertyChangeEvent object are set to null.
* object.
*/ */
INGEST_JOB_CANCELLED, INGEST_JOB_CANCELLED,
/** /**
@ -182,79 +253,84 @@ public class IngestManager {
}; };
/** /**
* Add property change listener to listen to ingest events. * Add an ingest event property change listener.
* *
* @param listener PropertyChangeListener to register * @param listener The PropertyChangeListener to register.
*/ */
public static void addPropertyChangeListener(final PropertyChangeListener listener) { public void addPropertyChangeListener(final PropertyChangeListener listener) {
pcs.addPropertyChangeListener(listener); pcs.addPropertyChangeListener(listener);
} }
public static void removePropertyChangeListener(final PropertyChangeListener listener) { /**
* Remove an ingest event property change listener.
*
* @param listener The PropertyChangeListener to unregister.
*/
public void removePropertyChangeListener(final PropertyChangeListener listener) {
pcs.removePropertyChangeListener(listener); pcs.removePropertyChangeListener(listener);
} }
static void fireIngestJobEvent(String eventType, long jobId) { /**
try { * Fire an ingest event signifying an ingest job started.
pcs.firePropertyChange(eventType, jobId, null); *
} catch (Exception e) { * @param ingestJobId The ingest job id.
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS */
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), void fireIngestJobStarted(long ingestJobId) {
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED, ingestJobId, null));
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event when file is done with a pipeline run * Fire an ingest event signifying an ingest job finished.
* *
* @param fileId ID of file that is done * @param ingestJobId The ingest job id.
*/ */
static void fireFileIngestDone(long fileId) { void fireIngestJobCompleted(long ingestJobId) {
try { fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event for ModuleDataEvent (when modules post data to blackboard, * Fire an ingest event signifying an ingest job was canceled.
* etc.)
* *
* @param moduleDataEvent * @param ingestJobId The ingest job id.
*/ */
static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { void fireIngestJobCancelled(long ingestJobId) {
try { fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null));
pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/** /**
* Fire event for ModuleContentChanged (when modules create new content that * Fire an ingest event signifying the ingest of a file is completed.
* needs to be analyzed)
* *
* @param moduleContentEvent * @param fileId The object id of file.
*/ */
static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { void fireFileIngestDone(long fileId) {
try { fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.FILE_DONE, fileId, null));
pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}
} }
/**
* Fire an event signifying a blackboard post by an ingest module.
*
* @param moduleDataEvent A ModuleDataEvent with the details of the posting.
*/
void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.DATA, moduleDataEvent, null));
}
/**
* Fire an event signifying discovery of additional content by an ingest
* module.
*
* @param moduleDataEvent A ModuleContentEvent with the details of the new
* content.
*/
void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.CONTENT_CHANGED, moduleContentEvent, null));
}
/**
* Post a message to the ingest messages in box.
*
* @param message The message to be posted.
*/
void postIngestMessage(IngestMessage message) { void postIngestMessage(IngestMessage message) {
if (ingestMessageBox != null) { if (ingestMessageBox != null) {
ingestMessageBox.displayMessage(message); ingestMessageBox.displayMessage(message);
@ -262,11 +338,10 @@ public class IngestManager {
} }
/** /**
* Get free disk space of a drive where ingest data are written to That * Get the free disk space of the drive where to which ingest data is being
* drive is being monitored by IngestMonitor thread when ingest is running. * written, as reported by the ingest monitor.
* Use this method to get amount of free disk space anytime.
* *
* @return amount of disk space, -1 if unknown * @return Free disk space, -1 if unknown // RJCTODO: What units?
*/ */
long getFreeDiskSpace() { long getFreeDiskSpace() {
if (ingestMonitor != null) { if (ingestMonitor != null) {
@ -276,33 +351,20 @@ public class IngestManager {
} }
} }
private void reportRunIngestModulesTaskDone(long taskId) { /**
ingestTasks.remove(taskId); * A Runnable that creates ingest jobs and submits the initial data source
* and file ingest tasks to the task schedulers.
*/
private class StartIngestJobsThread implements Runnable {
List<Long> completedJobs = new ArrayList<>(); private final long threadId;
for (IngestJob job : ingestJobs.values()) {
job.releaseIngestPipelinesForThread(taskId);
if (job.areIngestPipelinesShutDown() == true) {
completedJobs.add(job.getId());
}
}
for (Long jobId : completedJobs) {
IngestJob job = ingestJobs.remove(jobId);
fireEventTasksExecutor.submit(new FireIngestJobEventTask(jobId, job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED : IngestEvent.INGEST_JOB_COMPLETED));
}
}
private class StartIngestJobsTask implements Runnable {
private final long id;
private final List<Content> dataSources; private final List<Content> dataSources;
private final List<IngestModuleTemplate> moduleTemplates; private final List<IngestModuleTemplate> moduleTemplates;
private final boolean processUnallocatedSpace; private final boolean processUnallocatedSpace;
private ProgressHandle progress; private ProgressHandle progress;
StartIngestJobsTask(long taskId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) { StartIngestJobsThread(long threadId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
this.id = taskId; this.threadId = threadId;
this.dataSources = dataSources; this.dataSources = dataSources;
this.moduleTemplates = moduleTemplates; this.moduleTemplates = moduleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace; this.processUnallocatedSpace = processUnallocatedSpace;
@ -321,12 +383,16 @@ public class IngestManager {
"IngestManager.StartIngestJobsTask.run.cancelling", "IngestManager.StartIngestJobsTask.run.cancelling",
displayName)); displayName));
} }
IngestManager.getInstance().cancelIngestJobs(); cancelFileIngestThread(threadId);
return true; return true;
} }
}); });
progress.start(dataSources.size() * 2);
if (!ingestMonitor.isRunning()) {
ingestMonitor.start();
}
progress.start(2 * dataSources.size());
int workUnitsCompleted = 0; int workUnitsCompleted = 0;
for (Content dataSource : dataSources) { for (Content dataSource : dataSources) {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
@ -334,14 +400,7 @@ public class IngestManager {
} }
// Create an ingest job. // Create an ingest job.
IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace); List<IngestModuleError> errors = IngestJob.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace);
ingestJobs.put(ingestJob.getId(), ingestJob);
// Start at least one instance of each kind of ingest
// pipeline for this ingest job. This allows for an early out
// if the full ingest module lineup specified by the user
// cannot be started up.
List<IngestModuleError> errors = ingestJob.startUpIngestPipelines();
if (!errors.isEmpty()) { if (!errors.isEmpty()) {
// Report the error to the user. // Report the error to the user.
StringBuilder moduleStartUpErrors = new StringBuilder(); StringBuilder moduleStartUpErrors = new StringBuilder();
@ -367,169 +426,116 @@ public class IngestManager {
JOptionPane.showMessageDialog(null, notifyMessage.toString(), JOptionPane.showMessageDialog(null, notifyMessage.toString(),
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(),
"IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE);
// Jettison the ingest job and move on to the next one.
ingestJob.cancel();
ingestJobs.remove(ingestJob.getId());
break;
} }
// Queue the data source ingest tasks for the ingest job. fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED));
// Queue a data source ingest task for the ingest job.
final String inputName = dataSource.getName(); final String inputName = dataSource.getName();
progress.progress( progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg1", NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg1",
inputName), workUnitsCompleted); inputName), workUnitsCompleted);
scheduler.getDataSourceIngestScheduler().queueForIngest(ingestJob); DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(ingestJob, ingestJob.getDataSource()));
progress.progress( progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg2", NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg2",
inputName), ++workUnitsCompleted); inputName), ++workUnitsCompleted);
// Queue the file ingest tasks for the ingest job. // Queue the file ingest tasks for the ingest job.
progress.progress( progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg3", NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg3",
inputName), workUnitsCompleted); inputName), workUnitsCompleted);
scheduler.getFileIngestScheduler().queueForIngest(ingestJob); FileIngestTaskScheduler.getInstance().addTasks(ingestJob, ingestJob.getDataSource());
progress.progress( progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.progress.msg4", NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg4",
inputName), ++workUnitsCompleted); inputName), ++workUnitsCompleted);
if (!Thread.currentThread().isInterrupted()) { if (!Thread.currentThread().isInterrupted()) {
if (!ingestMonitor.isRunning()) { break;
ingestMonitor.start();
}
long taskId = ingestTaskId.incrementAndGet();
Future<?> task = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
int numberOfFileTasksRequested = getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfFileTasksRequested; ++i) {
taskId = ingestTaskId.incrementAndGet();
task = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId));
ingestTasks.put(taskId, task);
}
fireEventTasksExecutor.submit(new FireIngestJobEventTask(ingestJob.getId(), IngestEvent.INGEST_JOB_STARTED));
} }
} }
} catch (Exception ex) { } catch (Exception ex) {
String message = String.format("StartIngestJobsTask (id=%d) caught exception", id); //NON-NLS String message = String.format("StartIngestJobsTask (id=%d) caught exception", threadId); //NON-NLS
logger.log(Level.SEVERE, message, ex); logger.log(Level.SEVERE, message, ex);
MessageNotifyUtil.Message.error( MessageNotifyUtil.Message.error(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg")); NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg"));
} finally { } finally {
progress.finish(); progress.finish();
ingestTasks.remove(id); startIngestJobThreads.remove(threadId);
} }
} }
} }
private class RunDataSourceIngestModulesTask implements Runnable { /**
* A Runnable that acts as a consumer for the data ingest task scheduler's
private final long id; * task queue.
*/
RunDataSourceIngestModulesTask(long taskId) { private class DataSourceIngestThread implements Runnable {
id = taskId;
}
@Override @Override
public void run() { public void run() {
DataSourceIngestTaskScheduler scheduler = DataSourceIngestTaskScheduler.getInstance();
while (true) {
try { try {
IngestScheduler.DataSourceIngestScheduler scheduler = IngestScheduler.getInstance().getDataSourceIngestScheduler(); DataSourceIngestTask task = scheduler.getNextTask(); // Blocks.
IngestJob job = scheduler.getNextTask(); task.execute();
while (job != null) { } catch (InterruptedException ex) {
break;
}
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
break; break;
} }
job.getDataSourceIngestPipelineForThread(id).process();
job = scheduler.getNextTask();
}
} catch (Exception ex) {
String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
} }
} }
} }
private class RunFileSourceIngestModulesTask implements Runnable { /**
* A Runnable that acts as a consumer for the file task scheduler's task
private final long id; * queue.
*/
RunFileSourceIngestModulesTask(long taskId) { private static class FileIngestThread implements Runnable {
id = taskId;
}
@Override @Override
public void run() { public void run() {
FileIngestTaskScheduler scheduler = FileIngestTaskScheduler.getInstance();
while (true) {
try { try {
IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); FileIngestTask task = scheduler.getNextTask(); // Blocks.
FileIngestTask task = fileScheduler.getNextTask(); task.execute();
while (task != null) { } catch (InterruptedException ex) {
break;
}
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
break; break;
} }
IngestJob job = task.getJob();
job.updateFileTasksProgressBar(task.getFile().getName());
job.getFileIngestPipelineForThread(id).process(task.getFile());
task = fileScheduler.getNextTask();
}
} catch (Exception ex) {
String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id); //NON-NLS
logger.log(Level.SEVERE, message, ex);
} finally {
reportRunIngestModulesTaskDone(id);
} }
} }
} }
private class FireIngestJobEventTask implements Runnable { /**
* A Runnable that fire ingest events to ingest manager property change
* listeners.
*/
private class FireIngestEventThread implements Runnable {
private final long ingestJobId;
private final IngestEvent event; private final IngestEvent event;
private final Object oldValue;
private final Object newValue;
FireIngestJobEventTask(long ingestJobId, IngestEvent event) { FireIngestEventThread(IngestEvent event, Object oldValue, Object newValue) {
this.ingestJobId = ingestJobId;
this.event = event; this.event = event;
this.oldValue = oldValue;
this.newValue = newValue;
} }
@Override @Override
public void run() { public void run() {
fireIngestJobEvent(event.toString(), ingestJobId);
}
}
private class IngestCancellationWorker extends SwingWorker<Void, Void> {
@Override
protected Void doInBackground() throws Exception {
// First mark all of the ingest jobs as cancelled. This way the
// ingest modules will know they are being shut down due to
// cancellation when the cancelled run ingest module tasks release
// their pipelines.
for (IngestJob job : ingestJobs.values()) {
job.cancel();
}
for (Future<?> task : ingestTasks.values()) {
task.cancel(true);
}
// Jettision the remaining data source and file ingest tasks.
scheduler.getFileIngestScheduler().emptyQueues();
scheduler.getDataSourceIngestScheduler().emptyQueues();
return null;
}
@Override
protected void done() {
try { try {
super.get(); pcs.firePropertyChange(event.toString(), oldValue, newValue);
} catch (CancellationException | InterruptedException ex) { } catch (Exception e) {
} catch (Exception ex) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
logger.log(Level.SEVERE, "Error while cancelling ingest jobs", ex); //NON-NLS MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), // RJCTODO: Oddly named strings
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
} }
} }
} }

2
Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java
View File

@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content;
manager = IngestManager.getInstance(); manager = IngestManager.getInstance();
} }
try { try {
manager.cancelIngestJobs(); manager.cancelAllIngestJobs();
} finally { } finally {
//clear inbox //clear inbox
clearMessages(); clearMessages();

2
Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java
View File

@ -165,7 +165,7 @@ public final class IngestMonitor {
final String diskPath = root.getAbsolutePath(); final String diskPath = root.getAbsolutePath();
MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS
logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); //NON-NLS
manager.cancelIngestJobs(); manager.cancelAllIngestJobs();
IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage( IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage(
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath), NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath),
NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath))); NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath)));

744
Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java
View File

@ -1,744 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.VirtualDirectory;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM;
import org.sleuthkit.datamodel.TskData.TSK_FS_META_TYPE_ENUM;
/**
* Enqueues data source ingest and file ingest tasks for processing.
*/
final class IngestScheduler {
private static IngestScheduler instance;
private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName());
private final DataSourceIngestScheduler dataSourceIngestScheduler = new DataSourceIngestScheduler();
private final FileIngestScheduler fileIngestScheduler = new FileIngestScheduler();
private IngestScheduler() {
}
static synchronized IngestScheduler getInstance() {
if (instance == null) {
instance = new IngestScheduler();
}
return instance;
}
DataSourceIngestScheduler getDataSourceIngestScheduler() {
return dataSourceIngestScheduler;
}
FileIngestScheduler getFileIngestScheduler() {
return fileIngestScheduler;
}
static class FileIngestScheduler {
private TreeSet<FileIngestTask> rootDirectoryTasks;
private List<FileIngestTask> directoryTasks;
private LinkedList<FileIngestTask> fileTasks; //need to add to start and end quickly
private int filesEnqueuedEst = 0;
private int filesDequeued = 0;
private final static int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue()
| TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue()
| TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue()
| TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
private FileIngestScheduler() {
rootDirectoryTasks = new TreeSet<>(new RootTaskComparator());
directoryTasks = new ArrayList<>();
fileTasks = new LinkedList<>();
resetCounters();
}
private void resetCounters() {
filesEnqueuedEst = 0;
filesDequeued = 0;
}
@Override
public synchronized String toString() {
StringBuilder sb = new StringBuilder();
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.rootDirs.text")).append(rootDirectoryTasks.size());
for (FileIngestTask task : rootDirectoryTasks) {
sb.append(task.toString()).append(" ");
}
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curDirs.text")).append(directoryTasks.size());
for (FileIngestTask task : directoryTasks) {
sb.append(task.toString()).append(" ");
}
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.FileSched.toString.curFiles.text")).append(fileTasks.size());
for (FileIngestTask task : fileTasks) {
sb.append(task.toString()).append(" ");
}
return sb.toString();
}
synchronized void queueForIngest(IngestJob dataSourceTask) {
Content dataSource = dataSourceTask.getDataSource();
Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirVisitor());
List<AbstractFile> firstLevelFiles = new ArrayList<>();
if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
// The data source is file.
firstLevelFiles.add((AbstractFile) dataSource);
} else {
for (AbstractFile root : rootObjects) {
List<Content> children;
try {
children = root.getChildren();
if (children.isEmpty()) {
//add the root itself, could be unalloc file, child of volume or image
firstLevelFiles.add(root);
} else {
//root for fs root dir, schedule children dirs/files
for (Content child : children) {
if (child instanceof AbstractFile) {
firstLevelFiles.add((AbstractFile) child);
}
}
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS
}
}
}
for (AbstractFile firstLevelFile : firstLevelFiles) {
FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask);
if (shouldEnqueueTask(fileTask)) {
rootDirectoryTasks.add(fileTask);
}
}
// Update approx count of files to process in queues
filesEnqueuedEst = queryNumFilesinEnqueuedContents();
// Reshuffle/update the dir and file level queues if needed
updateQueues();
}
synchronized void queueFile(IngestJob ingestJob, AbstractFile file) {
FileIngestTask fileTask = new FileIngestTask(file, ingestJob);
if (shouldEnqueueTask(fileTask)) {
fileTasks.addFirst(fileTask);
++filesEnqueuedEst;
}
}
float getPercentageDone() {
if (filesEnqueuedEst == 0) {
return 0;
}
return ((100.f) * filesDequeued) / filesEnqueuedEst;
}
/**
* query num files enqueued total num of files to be enqueued.
*
* Counts all files for all the sources currently in the queues.
*
* @return approx. total num of files enqueued (or to be enqueued)
*/
private synchronized int queryNumFilesinEnqueuedContents() {
int totalFiles = 0;
List<Content> contents = this.getSourceContent();
final GetFilesCountVisitor countVisitor =
new GetFilesCountVisitor();
for (Content content : contents) {
totalFiles += content.accept(countVisitor);
}
logger.log(Level.INFO, "Total files to queue up: {0}", totalFiles); //NON-NLS
return totalFiles;
}
/**
* get total est. number of files to be enqueued for current ingest
* input sources in queues
*
* @return total number of files
*/
int getFilesEnqueuedEst() {
return filesEnqueuedEst;
}
/**
* Get number of files dequeued so far. This is reset after the same
* content is enqueued that is already in a queue
*
* @return number of files dequeued so far
*/
int getFilesDequeued() {
return filesDequeued;
}
synchronized FileIngestTask getNextTask() {
final FileIngestTask task = fileTasks.pollLast();
if (task != null) {
filesDequeued++;
updateQueues();
}
return task;
}
/**
* Shuffle the queues so that there are files in the files queue.
*
* @returns true if no more data in queue
*/
private synchronized void updateQueues() {
// we loop because we could have a directory that has all files
// that do not get enqueued
while (true) {
// There are files in the queue, we're done
if (this.fileTasks.isEmpty() == false) {
return;
}
// fill in the directory queue if it is empty.
if (this.directoryTasks.isEmpty()) {
// bail out if root is also empty -- we are done
if (rootDirectoryTasks.isEmpty()) {
return;
}
FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst();
directoryTasks.add(rootTask);
}
//pop and push AbstractFile directory children if any
//add the popped and its leaf children onto cur file list
FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1);
final AbstractFile parentFile = parentTask.file;
// add itself to the file list
if (shouldEnqueueTask(parentTask)) {
this.fileTasks.addLast(parentTask);
}
// add its children to the file and directory lists
try {
List<Content> children = parentFile.getChildren();
for (Content c : children) {
if (c instanceof AbstractFile) {
AbstractFile childFile = (AbstractFile) c;
FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getJob());
if (childFile.hasChildren()) {
this.directoryTasks.add(childTask);
} else if (shouldEnqueueTask(childTask)) {
this.fileTasks.addLast(childTask);
}
}
}
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Could not get children of file and update file queues: " //NON-NLS
+ parentFile.getName(), ex);
}
}
}
/**
* Return list of content objects that are in the queue to be processed.
*
* Helpful to determine whether ingest for particular input Content is
* active
*
* @return list of parent source content objects for files currently
* enqueued
*/
synchronized List<Content> getSourceContent() {
final Set<Content> contentSet = new HashSet<>();
for (FileIngestTask task : rootDirectoryTasks) {
contentSet.add(task.getJob().getDataSource());
}
for (FileIngestTask task : directoryTasks) {
contentSet.add(task.getJob().getDataSource());
}
for (FileIngestTask task : fileTasks) {
contentSet.add(task.getJob().getDataSource());
}
return new ArrayList<>(contentSet);
}
synchronized void emptyQueues() {
this.rootDirectoryTasks.clear();
this.directoryTasks.clear();
this.fileTasks.clear();
}
/**
* Check if the file is a special file that we should skip
*
* @param processTask a task whose file to check if should be queued of
* skipped
* @return true if should be enqueued, false otherwise
*/
private static boolean shouldEnqueueTask(final FileIngestTask processTask) {
final AbstractFile aFile = processTask.file;
//if it's unalloc file, skip if so scheduled
if (processTask.getJob().shouldProcessUnallocatedSpace() == false
&& aFile.getType().equals(TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS //unalloc files
)) {
return false;
}
String fileName = aFile.getName();
if (fileName.equals(".") || fileName.equals("..")) {
return false;
} else if (aFile instanceof org.sleuthkit.datamodel.File) {
final org.sleuthkit.datamodel.File f = (File) aFile;
//skip files in root dir, starting with $, containing : (not default attributes)
//with meta address < 32, i.e. some special large NTFS and FAT files
FileSystem fs = null;
try {
fs = f.getFileSystem();
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS
}
TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP;
if (fs != null) {
fsType = fs.getFsType();
}
if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) {
//not fat or ntfs, accept all files
return true;
}
boolean isInRootDir = false;
try {
isInRootDir = f.getParentDirectory().isRoot();
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS
}
if (isInRootDir && f.getMetaAddr() < 32) {
String name = f.getName();
if (name.length() > 0
&& name.charAt(0) == '$'
&& name.contains(":")) {
return false;
}
} else {
return true;
}
}
return true;
}
static class FileIngestTask {
private final AbstractFile file;
private final IngestJob task;
private FileIngestTask(AbstractFile file, IngestJob task) {
this.file = file;
this.task = task;
}
public IngestJob getJob() {
return task;
}
public AbstractFile getFile() {
return file;
}
@Override
public String toString() {
try {
return "ProcessTask{" + "file=" + file.getId() + ": " //NON-NLS
+ file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}';
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS
}
return "ProcessTask{" + "file=" + file.getId() + ": " //NON-NLS
+ file.getName() + '}';
}
/**
* two process tasks are equal when the file/dir and modules are the
* same this enables are not to queue up the same file/dir, modules
* tuples into the root dir set
*
* @param obj
* @return
*/
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final FileIngestTask other = (FileIngestTask) obj;
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
return false;
}
IngestJob thisTask = this.getJob();
IngestJob otherTask = other.getJob();
if (thisTask != otherTask
&& (thisTask == null || !thisTask.equals(otherTask))) {
return false;
}
return true;
}
@Override
public int hashCode() {
int hash = 5;
hash = 47 * hash + Objects.hashCode(this.file);
hash = 47 * hash + Objects.hashCode(this.task);
return hash;
}
}
/**
* Root dir sorter
*/
private static class RootTaskComparator implements Comparator<FileIngestTask> {
@Override
public int compare(FileIngestTask q1, FileIngestTask q2) {
AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file);
AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file);
if (p1 == p2) {
return (int) (q2.file.getId() - q1.file.getId());
} else {
return p2.ordinal() - p1.ordinal();
}
}
/**
* Priority determination for sorted AbstractFile, used by
* RootDirComparator
*/
private static class AbstractFilePriority {
enum Priority {
LAST, LOW, MEDIUM, HIGH
};
static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>();
static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>();
static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>();
static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>();
/* prioritize root directory folders based on the assumption that we are
* looking for user content. Other types of investigations may want different
* priorities. */
static {
// these files have no structure, so they go last
//unalloc files are handled as virtual files in getPriority()
//LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
//LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE));
LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
// orphan files are often corrupt and windows does not typically have
// user content, so put them towards the bottom
LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
// all other files go into the medium category too
MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
// user content is top priority
HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE));
HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE));
}
/**
* Get the scheduling priority for a given file.
*
* @param abstractFile
* @return
*/
static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) {
if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
//quickly filter out unstructured content
//non-fs virtual files and dirs, such as representing unalloc space
return AbstractFilePriority.Priority.LAST;
}
//determine the fs files priority by name
final String path = abstractFile.getName();
if (path == null) {
return AbstractFilePriority.Priority.MEDIUM;
}
for (Pattern p : HIGH_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.HIGH;
}
}
for (Pattern p : MEDIUM_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.MEDIUM;
}
}
for (Pattern p : LOW_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.LOW;
}
}
for (Pattern p : LAST_PRI_PATHS) {
Matcher m = p.matcher(path);
if (m.find()) {
return AbstractFilePriority.Priority.LAST;
}
}
//default is medium
return AbstractFilePriority.Priority.MEDIUM;
}
}
}
/**
* Get counts of ingestable files/dirs for the content input source.
*
* Note, also includes counts of all unalloc children files (for the fs,
* image, volume) even if ingest didn't ask for them
*/
static class GetFilesCountVisitor extends ContentVisitor.Default<Long> {
@Override
public Long visit(FileSystem fs) {
//recursion stop here
//case of a real fs, query all files for it
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
StringBuilder queryB = new StringBuilder();
queryB.append("( (fs_obj_id = ").append(fs.getId()); //NON-NLS
//queryB.append(") OR (fs_obj_id = NULL) )");
queryB.append(") )");
queryB.append(" AND ( (meta_type = ").append(TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue()); //NON-NLS
queryB.append(") OR (meta_type = ").append(TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_DIR.getValue()); //NON-NLS
queryB.append(" AND (name != '.') AND (name != '..')"); //NON-NLS
queryB.append(") )");
//queryB.append( "AND (type = ");
//queryB.append(TskData.TSK_DB_FILES_TYPE_ENUM.FS.getFileType());
//queryB.append(")");
try {
final String query = queryB.toString();
logger.log(Level.INFO, "Executing count files query: {0}", query); //NON-NLS
return sc.countFilesWhere(query);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); //NON-NLS
return 0L;
}
}
@Override
public Long visit(LayoutFile lf) {
//recursion stop here
//case of LayoutFile child of Image or Volume
return 1L;
}
private long getCountFromChildren(Content content) {
long count = 0;
try {
List<Content> children = content.getChildren();
if (children.size() > 0) {
for (Content child : children) {
count += child.accept(this);
}
} else {
count = 1;
}
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "Could not get count of objects from children to get num of total files to be ingested", ex); //NON-NLS
}
return count;
}
@Override
protected Long defaultVisit(Content cntnt) {
//recurse assuming this is image/vs/volume
//recursion stops at fs or unalloc file
return getCountFromChildren(cntnt);
}
}
/**
* Visitor that gets a collection of top level objects to be scheduled,
* such as root Dirs (if there is FS) or LayoutFiles and virtual
* directories, also if there is no FS.
*/
static class GetRootDirVisitor extends GetFilesContentVisitor {
@Override
public Collection<AbstractFile> visit(VirtualDirectory ld) {
//case when we hit a layout directoryor local file container, not under a real FS
//or when root virt dir is scheduled
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(ld);
return ret;
}
@Override
public Collection<AbstractFile> visit(LayoutFile lf) {
//case when we hit a layout file, not under a real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(lf);
return ret;
}
@Override
public Collection<AbstractFile> visit(Directory drctr) {
//we hit a real directory, a child of real FS
Collection<AbstractFile> ret = new ArrayList<>();
ret.add(drctr);
return ret;
}
@Override
public Collection<AbstractFile> visit(FileSystem fs) {
return getAllFromChildren(fs);
}
@Override
public Collection<AbstractFile> visit(File file) {
//can have derived files
return getAllFromChildren(file);
}
@Override
public Collection<AbstractFile> visit(DerivedFile derivedFile) {
//can have derived files
//TODO test this and overall scheduler with derived files
return getAllFromChildren(derivedFile);
}
@Override
public Collection<AbstractFile> visit(LocalFile localFile) {
//can have local files
//TODO test this and overall scheduler with local files
return getAllFromChildren(localFile);
}
}
}
static class DataSourceIngestScheduler {
private final LinkedList<IngestJob> tasks = new LinkedList<>();
private DataSourceIngestScheduler() {
}
synchronized void queueForIngest(IngestJob job) {
try {
if (job.getDataSource().getParent() != null) {
logger.log(Level.SEVERE, "Only parent-less Content (data sources) can be scheduled for DataSource ingest, skipping: {0}", job.getDataSource()); //NON-NLS
return;
}
} catch (TskCoreException e) {
logger.log(Level.SEVERE, "Error validating data source to be scheduled for DataSource ingest" + job.getDataSource(), e); //NON-NLS
return;
}
tasks.addLast(job);
}
public synchronized IngestJob getNextTask() {
return tasks.pollFirst();
}
synchronized void emptyQueues() {
tasks.clear();
}
synchronized int getCount() {
return tasks.size();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(NbBundle.getMessage(this.getClass(), "IngestScheduler.DataSourceScheduler.toString.size"))
.append(getCount());
for (IngestJob task : tasks) {
sb.append(task.toString()).append(" ");
}
return sb.toString();
}
}
}

4
Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java
View File

@ -95,7 +95,7 @@ public final class IngestServices {
* artifact data * artifact data
*/ */
public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
IngestManager.fireModuleDataEvent(moduleDataEvent); IngestManager.fireIngestModuleDataEvent(moduleDataEvent);
} }
/** /**
@ -107,7 +107,7 @@ public final class IngestServices {
* changed * changed
*/ */
public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
IngestManager.fireModuleContentEvent(moduleContentEvent); IngestManager.fireIngestModuleContentEvent(moduleContentEvent);
} }
/** /**