diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED index 9a43ffe229..528d3a5088 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED @@ -247,10 +247,15 @@ AddImageWizardIngestConfigPanel.dsProcDone.errs.text=*Errors encountered in addi AddImageWizardIngestConfigVisual.getName.text=Configure Ingest AddImageWizardIterator.stepXofN=Step {0} of {1} AddLocalFilesTask.localFileAdd.progress.text=Adding: {0}/{1} -Case.getCurCase.exception.noneOpen=Cannot get the current case; there is no case open! +Case.getCurCase.exception.noneOpen=Cannot get the current case; there is no case open\! Case.open.msgDlg.updated.msg=Updated case database schema.\nA backup copy of the database with the following path has been made:\n {0} Case.open.msgDlg.updated.title=Case Database Schema Update -Case.checkImgExist.confDlg.doesntExist.msg=One of the images associated with \nthis case are missing. Would you like to search for them now?\nPreviously, the image was located at:\n{0}\nPlease note that you will still be able to browse directories and generate reports\nif you choose No, but you will not be able to view file content or run the ingest process. +Case.checkImgExist.confDlg.doesntExist.msg=One of the images associated with \n\ +this case are missing. Would you like to search for them now?\n\ +Previously, the image was located at:\n\ +{0}\n\ +Please note that you will still be able to browse directories and generate reports\n\ +if you choose No, but you will not be able to view file content or run the ingest process. Case.checkImgExist.confDlg.doesntExist.title=Missing Image Case.addImg.exception.msg=Error adding image to the case Case.updateCaseName.exception.msg=Error while trying to update the case name. @@ -269,9 +274,12 @@ Case.GetCaseTypeGivenPath.Failure=Unable to get case type Case.metaDataFileCorrupt.exception.msg=The case metadata file (.aut) is corrupted. Case.deleteReports.deleteFromDiskException.log.msg=Unable to delete the report from the disk. Case.deleteReports.deleteFromDiskException.msg=Unable to delete the report {0} from the disk.\nYou may manually delete it from {1} -CaseDeleteAction.closeConfMsg.text=Are you sure want to close and delete this case? \nCase Name: {0}\nCase Directory: {1} +CaseDeleteAction.closeConfMsg.text=Are you sure want to close and delete this case? \n\ + Case Name: {0}\n\ + Case Directory: {1} CaseDeleteAction.closeConfMsg.title=Warning: Closing the Current Case -CaseDeleteAction.msgDlg.fileInUse.msg=The delete action cannot be fully completed because the folder or file in it is open by another program.\n\nClose the folder and file and try again or you can delete the case manually. +CaseDeleteAction.msgDlg.fileInUse.msg=The delete action cannot be fully completed because the folder or file in it is open by another program.\n\n\ +Close the folder and file and try again or you can delete the case manually. CaseDeleteAction.msgDlg.fileInUse.title=Error: Folder In Use CaseDeleteAction.msgDlg.caseDelete.msg=Case {0} has been deleted. CaseOpenAction.autFilter.title={0} Case File ( {1}) @@ -303,7 +311,8 @@ NewCaseWizardAction.databaseProblem1.text=Cannot open database. Cancelling case NewCaseWizardAction.databaseProblem2.text=Error NewCaseWizardPanel1.validate.errMsg.invalidSymbols=The Case Name cannot contain any of the following symbols: \\ / : * ? " < > | NewCaseWizardPanel1.validate.errMsg.dirExists=Case directory ''{0}'' already exists. -NewCaseWizardPanel1.validate.confMsg.createDir.msg=The base directory "{0}" does not exist. \n\nDo you want to create that directory? +NewCaseWizardPanel1.validate.confMsg.createDir.msg=The base directory "{0}" does not exist. \n\n\ + Do you want to create that directory? NewCaseWizardPanel1.validate.confMsg.createDir.title=Create directory NewCaseWizardPanel1.validate.errMsg.cantCreateParDir.msg=Error: Could not create case parent directory {0} NewCaseWizardPanel1.validate.errMsg.prevCreateBaseDir.msg=Prevented from creating base directory {0} @@ -332,6 +341,7 @@ OptionalCasePropertiesPanel.lbPointOfContactPhoneLabel.text=Phone: OptionalCasePropertiesPanel.orgainizationPanel.border.title=Organization RecentCases.exception.caseIdxOutOfRange.msg=Recent case index {0} is out of range. RecentCases.getName.text=Clear Recent Cases +# {0} - case name RecentItems.openRecentCase.msgDlg.text=Case {0} no longer exists. SelectDataSourceProcessorPanel.name.text=Select Data Source Type StartupWindow.title.text=Welcome @@ -344,6 +354,7 @@ StartupWindowProvider.openCase.noFile=Unable to open previously open case becaus UnpackagePortableCaseDialog.title.text=Unpackage Portable Case UnpackagePortableCaseDialog.UnpackagePortableCaseDialog.extensions=Portable case package (.zip, .zip.001) UnpackagePortableCaseDialog.validatePaths.badExtension=File extension must be .zip or .zip.001 +# {0} - case folder UnpackagePortableCaseDialog.validatePaths.caseFolderExists=Folder {0} already exists UnpackagePortableCaseDialog.validatePaths.caseIsNotFile=Selected path is not a file UnpackagePortableCaseDialog.validatePaths.caseNotFound=File does not exist @@ -358,8 +369,8 @@ UnpackageWorker.doInBackground.previouslySeenCase=Case has been previously opene UpdateRecentCases.menuItem.clearRecentCases.text=Clear Recent Cases UpdateRecentCases.menuItem.empty=-Empty- AddImageWizardIngestConfigPanel.CANCEL_BUTTON.text=Cancel -NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on "C:" drive -NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on "C:" drive. Case folder is created on the target system +NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on \"C:\" drive +NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on \"C:\" drive. Case folder is created on the target system NewCaseVisualPanel1.CaseFolderOnInternalDriveLinuxError.text=Warning: Path to case folder is on the target system. Create case folder in mounted drive. NewCaseVisualPanel1.uncPath.error=Error: UNC paths are not allowed for Single-User cases CollaborationMonitor.addingDataSourceStatus.msg={0} adding data source @@ -367,7 +378,7 @@ CollaborationMonitor.analyzingDataSourceStatus.msg={0} analyzing {1} MissingImageDialog.lbWarning.text= MissingImageDialog.lbWarning.toolTipText= NewCaseVisualPanel1.caseParentDirWarningLabel.text= -NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User\t\t +NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-User NewCaseVisualPanel1.caseTypeLabel.text=Case Type: SingleUserCaseConverter.BadDatabaseFileName=Database file does not exist! diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 417239c99d..6f10bcf364 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -530,25 +530,6 @@ public class Case { eventPublisher.publish(new TimelineEventAddedEvent(event)); } - @SuppressWarnings("deprecation") - @Subscribe - public void publishArtifactsPostedEvent(Blackboard.ArtifactsPostedEvent event) { - for (BlackboardArtifact.Type artifactType : event.getArtifactTypes()) { - /* - * IngestServices.fireModuleDataEvent is deprecated to - * discourage ingest module writers from using it (they should - * use org.sleuthkit.datamodel.Blackboard.postArtifact(s) - * instead), but a way to publish - * Blackboard.ArtifactsPostedEvents from the SleuthKit layer as - * Autopsy ModuleDataEvents is still needed. - */ - IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent( - event.getModuleName(), - artifactType, - event.getArtifacts(artifactType))); - } - } - @Subscribe public void publishOsAccountsAddedEvent(TskEvent.OsAccountsAddedTskEvent event) { hasData = true; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java new file mode 100755 index 0000000000..0ed9f53518 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoDataArtifactIngestModule.java @@ -0,0 +1,54 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import java.util.concurrent.atomic.AtomicLong; +import org.sleuthkit.autopsy.ingest.DataArtifactIngestModule; +import org.sleuthkit.autopsy.ingest.IngestMessage; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.datamodel.DataArtifact; + +/** + * RJCTODO + * + * NOTE TO REVIEWER: + * + * This is a placeholder data artifact ingest module that counts the number of + * data artifacts it processes and posts the final count to the ingest inbox. + * The guts of the module will be supplied by a later PR. + */ +public class CentralRepoDataArtifactIngestModule implements DataArtifactIngestModule { + + private final AtomicLong artifactCounter = new AtomicLong(); + + @Override + public ProcessResult process(DataArtifact artifact) { + artifactCounter.incrementAndGet(); + return ProcessResult.OK; + } + + @Override + public void shutDown() { + IngestServices.getInstance().postMessage(IngestMessage.createMessage( + IngestMessage.MessageType.INFO, + CentralRepoIngestModuleFactory.getModuleName(), + String.format("%d data artifacts processed", artifactCounter.get()))); //NON-NLS + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java index 1c34f1ffad..39c80abefc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModuleFactory.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2018 Basis Technology Corp. + * Copyright 2015-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,15 +26,18 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.DataArtifactIngestModule; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** - * Factory for Central Repository ingest modules + * Factory for Central Repository ingest modules. */ @ServiceProvider(service = org.sleuthkit.autopsy.ingest.IngestModuleFactory.class) -@NbBundle.Messages({"CentralRepoIngestModuleFactory.ingestmodule.name=Central Repository", - "CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) +@NbBundle.Messages({ + "CentralRepoIngestModuleFactory.ingestmodule.name=Central Repository", + "CentralRepoIngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation" +}) public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { /** @@ -61,7 +64,7 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { return Version.getVersion(); } - @Override + @Override public boolean isFileIngestModuleFactory() { return true; } @@ -72,15 +75,15 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { return new CentralRepoIngestModule((IngestSettings) settings); } /* - * Compatibility check for older versions. + * Earlier versions of the modules had no ingest job settings. Create a + * module with the default settings. */ if (settings instanceof NoIngestModuleIngestJobSettings) { - return new CentralRepoIngestModule(new IngestSettings()); + return new CentralRepoIngestModule((IngestSettings) getDefaultIngestJobSettings()); } - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } - + @Override public boolean hasGlobalSettingsPanel() { return true; @@ -92,7 +95,7 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } - + @Override public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { return new IngestSettings(); @@ -109,13 +112,23 @@ public class CentralRepoIngestModuleFactory extends IngestModuleFactoryAdapter { return new IngestSettingsPanel((IngestSettings) settings); } /* - * Compatibility check for older versions. + * Earlier versions of the modules had no ingest job settings. Create a + * panel with the default settings. */ if (settings instanceof NoIngestModuleIngestJobSettings) { - return new IngestSettingsPanel(new IngestSettings()); + return new IngestSettingsPanel((IngestSettings) getDefaultIngestJobSettings()); } - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } + @Override + public boolean isDataArtifactIngestModuleFactory() { + return true; + } + + @Override + public DataArtifactIngestModule createDataArtifactIngestModule(IngestModuleIngestJobSettings settings) { + return new CentralRepoDataArtifactIngestModule(); + } + } diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED index 90e7cc4a11..447b4e3038 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/Bundle.properties-MERGED @@ -2,7 +2,7 @@ CommandLineIngestSettingPanel_empty_report_name_mgs=Report profile name was empt CommandLineIngestSettingPanel_existing_report_name_mgs=Report profile name was already exists, no profile created. CommandListIngestSettingsPanel_Default_Report_DisplayName=Default CommandListIngestSettingsPanel_Make_Config=Make new profile... -CommandListIngestSettingsPanel_Report_Name_Msg=Please supply a report profile name: +CommandListIngestSettingsPanel_Report_Name_Msg=Please supply a report profile name (commas not allowed): OpenIDE-Module-Name=CommandLineAutopsy OptionsCategory_Keywords_Command_Line_Ingest_Settings=Command Line Ingest Settings OptionsCategory_Keywords_General=Options diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java index edbcd7a76f..635e5dfa6d 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2019-2020 Basis Technology Corp. + * Copyright 2019-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,7 +62,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleError; import org.sleuthkit.autopsy.ingest.IngestProfiles; import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; import org.sleuthkit.autopsy.modules.interestingitems.FilesSetsManager; -import org.sleuthkit.autopsy.progress.LoggingProgressIndicator; import org.sleuthkit.autopsy.report.infrastructure.ReportGenerator; import org.sleuthkit.autopsy.report.infrastructure.ReportProgressIndicator; import org.sleuthkit.datamodel.Content; @@ -73,7 +72,7 @@ import org.sleuthkit.datamodel.TskCoreException; * cause Autopsy to create a case, add a specified data source, run ingest on * that data source, list all data sources in the case, and generate reports. */ -public class CommandLineIngestManager extends CommandLineManager{ +public class CommandLineIngestManager extends CommandLineManager { private static final Logger LOGGER = Logger.getLogger(CommandLineIngestManager.class.getName()); private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.CANCELLED, IngestManager.IngestJobEvent.COMPLETED); @@ -152,161 +151,161 @@ public class CommandLineIngestManager extends CommandLineManager{ switch (type) { case CREATE_CASE: try { - LOGGER.log(Level.INFO, "Processing 'Create Case' command"); - System.out.println("Processing 'Create Case' command"); - Map inputs = command.getInputs(); - String baseCaseName = inputs.get(CommandLineCommand.InputType.CASE_NAME.name()); - String rootOutputDirectory = inputs.get(CommandLineCommand.InputType.CASES_BASE_DIR_PATH.name()); - CaseType caseType = CaseType.SINGLE_USER_CASE; - String caseTypeString = inputs.get(CommandLineCommand.InputType.CASE_TYPE.name()); - if (caseTypeString != null && caseTypeString.equalsIgnoreCase(CommandLineOptionProcessor.CASETYPE_MULTI)) { - caseType = CaseType.MULTI_USER_CASE; - } - openCase(baseCaseName, rootOutputDirectory, caseType); - - String outputDirPath = getOutputDirPath(caseForJob); - OutputGenerator.saveCreateCaseOutput(caseForJob, outputDirPath, baseCaseName); - } catch (CaseActionException ex) { - String baseCaseName = command.getInputs().get(CommandLineCommand.InputType.CASE_NAME.name()); - LOGGER.log(Level.SEVERE, "Error creating or opening case " + baseCaseName, ex); - System.out.println("Error creating or opening case " + baseCaseName); - // Do not process any other commands - return; + LOGGER.log(Level.INFO, "Processing 'Create Case' command"); + System.out.println("Processing 'Create Case' command"); + Map inputs = command.getInputs(); + String baseCaseName = inputs.get(CommandLineCommand.InputType.CASE_NAME.name()); + String rootOutputDirectory = inputs.get(CommandLineCommand.InputType.CASES_BASE_DIR_PATH.name()); + CaseType caseType = CaseType.SINGLE_USER_CASE; + String caseTypeString = inputs.get(CommandLineCommand.InputType.CASE_TYPE.name()); + if (caseTypeString != null && caseTypeString.equalsIgnoreCase(CommandLineOptionProcessor.CASETYPE_MULTI)) { + caseType = CaseType.MULTI_USER_CASE; } - break; + openCase(baseCaseName, rootOutputDirectory, caseType); + + String outputDirPath = getOutputDirPath(caseForJob); + OutputGenerator.saveCreateCaseOutput(caseForJob, outputDirPath, baseCaseName); + } catch (CaseActionException ex) { + String baseCaseName = command.getInputs().get(CommandLineCommand.InputType.CASE_NAME.name()); + LOGGER.log(Level.SEVERE, "Error creating or opening case " + baseCaseName, ex); + System.out.println("Error creating or opening case " + baseCaseName); + // Do not process any other commands + return; + } + break; case ADD_DATA_SOURCE: try { - LOGGER.log(Level.INFO, "Processing 'Add Data Source' command"); - System.out.println("Processing 'Add Data Source' command"); - Map inputs = command.getInputs(); + LOGGER.log(Level.INFO, "Processing 'Add Data Source' command"); + System.out.println("Processing 'Add Data Source' command"); + Map inputs = command.getInputs(); - // open the case, if it hasn't been already opened by CREATE_CASE command - if (caseForJob == null) { - String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); - caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); - } - - String dataSourcePath = inputs.get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); - dataSource = new AutoIngestDataSource("", Paths.get(dataSourcePath)); - runDataSourceProcessor(caseForJob, dataSource); - - String outputDirPath = getOutputDirPath(caseForJob); - OutputGenerator.saveAddDataSourceOutput(caseForJob, dataSource, outputDirPath); - } catch (InterruptedException | AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException | CaseActionException ex) { - String dataSourcePath = command.getInputs().get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); - LOGGER.log(Level.SEVERE, "Error adding data source " + dataSourcePath, ex); - System.out.println("Error adding data source " + dataSourcePath); - // Do not process any other commands - return; + // open the case, if it hasn't been already opened by CREATE_CASE command + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); } - break; + + String dataSourcePath = inputs.get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); + dataSource = new AutoIngestDataSource("", Paths.get(dataSourcePath)); + runDataSourceProcessor(caseForJob, dataSource); + + String outputDirPath = getOutputDirPath(caseForJob); + OutputGenerator.saveAddDataSourceOutput(caseForJob, dataSource, outputDirPath); + } catch (InterruptedException | AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException | CaseActionException ex) { + String dataSourcePath = command.getInputs().get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); + LOGGER.log(Level.SEVERE, "Error adding data source " + dataSourcePath, ex); + System.out.println("Error adding data source " + dataSourcePath); + // Do not process any other commands + return; + } + break; case RUN_INGEST: try { - LOGGER.log(Level.INFO, "Processing 'Run Ingest' command"); - System.out.println("Processing 'Run Ingest' command"); - Map inputs = command.getInputs(); + LOGGER.log(Level.INFO, "Processing 'Run Ingest' command"); + System.out.println("Processing 'Run Ingest' command"); + Map inputs = command.getInputs(); - // open the case, if it hasn't been already opened by CREATE_CASE or ADD_DATA_SOURCE commands - if (caseForJob == null) { - String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); - caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); - } - - // populate the AutoIngestDataSource structure, if that hasn't been done by ADD_DATA_SOURCE command - if (dataSource == null) { - - String dataSourceId = inputs.get(CommandLineCommand.InputType.DATA_SOURCE_ID.name()); - Long dataSourceObjId = Long.valueOf(dataSourceId); - - // get Content object for the data source - Content content = null; - try { - content = Case.getCurrentCaseThrows().getSleuthkitCase().getContentById(dataSourceObjId); - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Exception while trying to find data source with object ID " + dataSourceId, ex); - System.out.println("Exception while trying to find data source with object ID " + dataSourceId); - // Do not process any other commands - return; - } - - if (content == null) { - LOGGER.log(Level.SEVERE, "Unable to find data source with object ID {0}", dataSourceId); - System.out.println("Unable to find data source with object ID " + dataSourceId); - // Do not process any other commands - return; - } - - // populate the AutoIngestDataSource structure - dataSource = new AutoIngestDataSource("", Paths.get(content.getName())); - List contentList = Arrays.asList(new Content[]{content}); - List errorList = new ArrayList<>(); - dataSource.setDataSourceProcessorOutput(NO_ERRORS, errorList, contentList); - } - - // run ingest - String ingestProfile = inputs.get(CommandLineCommand.InputType.INGEST_PROFILE_NAME.name()); - analyze(dataSource, ingestProfile); - } catch (InterruptedException | CaseActionException ex) { - String dataSourcePath = command.getInputs().get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); - LOGGER.log(Level.SEVERE, "Error running ingest on data source " + dataSourcePath, ex); - System.out.println("Error running ingest on data source " + dataSourcePath); - // Do not process any other commands - return; + // open the case, if it hasn't been already opened by CREATE_CASE or ADD_DATA_SOURCE commands + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); } - break; + + // populate the AutoIngestDataSource structure, if that hasn't been done by ADD_DATA_SOURCE command + if (dataSource == null) { + + String dataSourceId = inputs.get(CommandLineCommand.InputType.DATA_SOURCE_ID.name()); + Long dataSourceObjId = Long.valueOf(dataSourceId); + + // get Content object for the data source + Content content = null; + try { + content = Case.getCurrentCaseThrows().getSleuthkitCase().getContentById(dataSourceObjId); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Exception while trying to find data source with object ID " + dataSourceId, ex); + System.out.println("Exception while trying to find data source with object ID " + dataSourceId); + // Do not process any other commands + return; + } + + if (content == null) { + LOGGER.log(Level.SEVERE, "Unable to find data source with object ID {0}", dataSourceId); + System.out.println("Unable to find data source with object ID " + dataSourceId); + // Do not process any other commands + return; + } + + // populate the AutoIngestDataSource structure + dataSource = new AutoIngestDataSource("", Paths.get(content.getName())); + List contentList = Arrays.asList(new Content[]{content}); + List errorList = new ArrayList<>(); + dataSource.setDataSourceProcessorOutput(NO_ERRORS, errorList, contentList); + } + + // run ingest + String ingestProfile = inputs.get(CommandLineCommand.InputType.INGEST_PROFILE_NAME.name()); + analyze(dataSource, ingestProfile); + } catch (InterruptedException | CaseActionException ex) { + String dataSourcePath = command.getInputs().get(CommandLineCommand.InputType.DATA_SOURCE_PATH.name()); + LOGGER.log(Level.SEVERE, "Error running ingest on data source " + dataSourcePath, ex); + System.out.println("Error running ingest on data source " + dataSourcePath); + // Do not process any other commands + return; + } + break; case LIST_ALL_DATA_SOURCES: try { - LOGGER.log(Level.INFO, "Processing 'List All Data Sources' command"); - System.out.println("Processing 'List All Data Sources' command"); - Map inputs = command.getInputs(); + LOGGER.log(Level.INFO, "Processing 'List All Data Sources' command"); + System.out.println("Processing 'List All Data Sources' command"); + Map inputs = command.getInputs(); - // open the case, if it hasn't been already opened by previous command - if (caseForJob == null) { - String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); - caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); - } - - String outputDirPath = getOutputDirPath(caseForJob); - OutputGenerator.listAllDataSources(caseForJob, outputDirPath); - } catch (CaseActionException ex) { - String caseDirPath = command.getInputs().get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); - LOGGER.log(Level.SEVERE, "Error opening case in case directory: " + caseDirPath, ex); - System.out.println("Error opening case in case directory: " + caseDirPath); - // Do not process any other commands - return; + // open the case, if it hasn't been already opened by previous command + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); } - break; + + String outputDirPath = getOutputDirPath(caseForJob); + OutputGenerator.listAllDataSources(caseForJob, outputDirPath); + } catch (CaseActionException ex) { + String caseDirPath = command.getInputs().get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + LOGGER.log(Level.SEVERE, "Error opening case in case directory: " + caseDirPath, ex); + System.out.println("Error opening case in case directory: " + caseDirPath); + // Do not process any other commands + return; + } + break; case GENERATE_REPORTS: try { - LOGGER.log(Level.INFO, "Processing 'Generate Reports' command"); - System.out.println("Processing 'Generate Reports' command"); - Map inputs = command.getInputs(); + LOGGER.log(Level.INFO, "Processing 'Generate Reports' command"); + System.out.println("Processing 'Generate Reports' command"); + Map inputs = command.getInputs(); - // open the case, if it hasn't been already opened by previous command - if (caseForJob == null) { - String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); - caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); - } - // generate reports - String reportName = inputs.get(CommandLineCommand.InputType.REPORT_PROFILE_NAME.name()); - if (reportName == null) { - reportName = CommandLineIngestSettingsPanel.getDefaultReportingConfigName(); - } - - // generate reports - ReportProgressIndicator progressIndicator = new ReportProgressIndicator(new CommandLineProgressIndicator()); - ReportGenerator generator = new ReportGenerator(reportName, progressIndicator); - generator.generateReports(); - } catch (CaseActionException ex) { - String caseDirPath = command.getInputs().get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); - LOGGER.log(Level.SEVERE, "Error opening case in case directory: " + caseDirPath, ex); - System.out.println("Error opening case in case directory: " + caseDirPath); - // Do not process any other commands - return; + // open the case, if it hasn't been already opened by previous command + if (caseForJob == null) { + String caseDirPath = inputs.get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + caseForJob = CommandLineIngestManager.this.openCase(caseDirPath); } - break; + // generate reports + String reportName = inputs.get(CommandLineCommand.InputType.REPORT_PROFILE_NAME.name()); + if (reportName == null) { + reportName = CommandLineIngestSettingsPanel.getDefaultReportingConfigName(); + } + + // generate reports + ReportProgressIndicator progressIndicator = new ReportProgressIndicator(new CommandLineProgressIndicator()); + ReportGenerator generator = new ReportGenerator(reportName, progressIndicator); + generator.generateReports(); + } catch (CaseActionException ex) { + String caseDirPath = command.getInputs().get(CommandLineCommand.InputType.CASE_FOLDER_PATH.name()); + LOGGER.log(Level.SEVERE, "Error opening case in case directory: " + caseDirPath, ex); + System.out.println("Error opening case in case directory: " + caseDirPath); + // Do not process any other commands + return; + } + break; default: break; } @@ -381,10 +380,13 @@ public class CommandLineIngestManager extends CommandLineManager{ * @param dataSource The data source. * * @throws AutoIngestDataSourceProcessorException if there was a DSP - * processing error. + * processing error. * - * @throws InterruptedException running the job processing task while - * blocking, i.e., if auto ingest is shutting down. + * @throws InterruptedException running the job + * processing task while + * blocking, i.e., if + * auto ingest is + * shutting down. */ private void runDataSourceProcessor(Case caseForJob, AutoIngestDataSource dataSource) throws InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java index db60e7a0af..ca11d03f13 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestSettingsPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2019-2020 Basis Technology Corp. + * Copyright 2019-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -280,7 +280,7 @@ public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { add(nodePanel, java.awt.BorderLayout.CENTER); }// //GEN-END:initComponents @Messages({ - "CommandListIngestSettingsPanel_Report_Name_Msg=Please supply a report profile name:", + "CommandListIngestSettingsPanel_Report_Name_Msg=Please supply a report profile name (commas not allowed):", "CommandLineIngestSettingPanel_empty_report_name_mgs=Report profile name was empty, no profile created.", "CommandLineIngestSettingPanel_existing_report_name_mgs=Report profile name was already exists, no profile created." }) @@ -288,6 +288,10 @@ public class CommandLineIngestSettingsPanel extends javax.swing.JPanel { String reportName = getReportName(); if (reportName.equals(Bundle.CommandListIngestSettingsPanel_Make_Config())) { reportName = JOptionPane.showInputDialog(this, Bundle.CommandListIngestSettingsPanel_Report_Name_Msg()); + + // sanitize report name. Remove all commas because in CommandLineOptionProcessor we use commas + // to separate multiple report names + reportName = reportName.replaceAll(",", ""); // User hit cancel if (reportName == null) { diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java index ad84bdcd0f..2f5644928e 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2019-2020 Basis Technology Corp. + * Copyright 2019-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,12 +20,15 @@ package org.sleuthkit.autopsy.commandlineingest; import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.sleuthkit.autopsy.coreutils.Logger; import org.netbeans.api.sendopts.CommandException; import org.netbeans.spi.sendopts.Env; @@ -291,7 +294,6 @@ public class CommandLineOptionProcessor extends OptionProcessor { } // Add "GENERATE_REPORTS" command, if present - String reportProfile = null; if (values.containsKey(generateReportsOption)) { // 'caseDir' must only be specified if the case is not being created during the current run @@ -300,24 +302,34 @@ public class CommandLineOptionProcessor extends OptionProcessor { handleError("'caseDir' argument is empty"); } + List reportProfiles; argDirs = values.get(generateReportsOption); if (argDirs.length > 0) { - reportProfile = argDirs[0]; + // use custom report configuration(s) + reportProfiles = Stream.of(argDirs[0].split(",")) + .map(String::trim) + .collect(Collectors.toList()); + + if (reportProfiles == null || reportProfiles.isEmpty()) { + handleError("'generateReports' argument is empty"); + } + + for (String reportProfile : reportProfiles) { + if (reportProfile.isEmpty()) { + handleError("Empty report profile name"); + } + CommandLineCommand newCommand = new CommandLineCommand(CommandLineCommand.CommandType.GENERATE_REPORTS); + newCommand.addInputValue(CommandLineCommand.InputType.CASE_FOLDER_PATH.name(), caseDir); + newCommand.addInputValue(CommandLineCommand.InputType.REPORT_PROFILE_NAME.name(), reportProfile); + commands.add(newCommand); + } + } else { + // use default report configuration + CommandLineCommand newCommand = new CommandLineCommand(CommandLineCommand.CommandType.GENERATE_REPORTS); + newCommand.addInputValue(CommandLineCommand.InputType.CASE_FOLDER_PATH.name(), caseDir); + commands.add(newCommand); } - // If the user doesn't supply an options for generateReports the - // argsDirs length will be 0, so if reportProfile is empty - // something is not right. - if (reportProfile != null && reportProfile.isEmpty()) { - handleError("'generateReports' argument is empty"); - } - - CommandLineCommand newCommand = new CommandLineCommand(CommandLineCommand.CommandType.GENERATE_REPORTS); - newCommand.addInputValue(CommandLineCommand.InputType.CASE_FOLDER_PATH.name(), caseDir); - if (reportProfile != null) { - newCommand.addInputValue(CommandLineCommand.InputType.REPORT_PROFILE_NAME.name(), reportProfile); - } - commands.add(newCommand); runFromCommandLine = true; } } diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java index 4b2b4ac972..a45d96046a 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountViewer.java @@ -141,7 +141,7 @@ public class OsAccountViewer extends javax.swing.JPanel implements DataContentVi @Override public int isPreferred(Node node) { - return 5; + return 1; } /** diff --git a/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED index 51f1208f61..1d50092e80 100755 --- a/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/core/Bundle.properties-MERGED @@ -3,7 +3,13 @@ Installer.closing.confirmationDialog.title=Ingest is Running # {0} - exception message Installer.closing.messageBox.caseCloseExceptionMessage=Error closing case: {0} OpenIDE-Module-Display-Category=Infrastructure -OpenIDE-Module-Long-Description=This is the core Autopsy module.\n\nThe module contains the core components needed for the bare application to run; the RCP platform, windowing GUI, sleuthkit bindings, datamodel / storage, explorer, result viewers, content viewers, ingest framework, reporting, and core tools, such as the file search.\n\nThe framework included in the module contains APIs for developing modules for ingest, viewers and reporting. The modules can be deployed as Plugins using the Autopsy plugin installer.\nThis module should not be uninstalled - without it, Autopsy will not run.\n\nFor more information, see http://www.sleuthkit.org/autopsy/ +OpenIDE-Module-Long-Description=\ + This is the core Autopsy module.\n\n\ + The module contains the core components needed for the bare application to run; the RCP platform, windowing GUI, sleuthkit bindings, datamodel / storage, explorer, result viewers, content viewers, ingest framework, reporting, and core tools, such as the file search.\n\n\ + The framework included in the module contains APIs for developing modules for ingest, viewers and reporting. \ + The modules can be deployed as Plugins using the Autopsy plugin installer.\n\ + This module should not be uninstalled - without it, Autopsy will not run.\n\n\ + For more information, see http://www.sleuthkit.org/autopsy/ OpenIDE-Module-Name=Autopsy-Core OpenIDE-Module-Short-Description=Autopsy Core Module org_sleuthkit_autopsy_core_update_center=http://sleuthkit.org/autopsy/updates.xml diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED index 609d68bcd1..0636340b0b 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED @@ -75,9 +75,9 @@ DataContentViewerHex.totalPageLabel.text_1=100 DataContentViewerHex.pageLabel2.text=Page # Product Information panel -LBL_Description=
\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
+LBL_Description=
\n Product Version: {0} ({9})
Sleuth Kit Version: {7}
Netbeans RCP Build: {8}
Java: {1}; {2}
System: {3}; {4}; {5}
Userdir: {6}
Format_OperatingSystem_Value={0} version {1} running on {2} -LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
+LBL_Copyright=
Autopsy™ is a digital forensics platform based on The Sleuth Kit™ and other tools.
Copyright © 2003-2020.
SortChooser.dialogTitle=Choose Sort Criteria ThumbnailViewChildren.progress.cancelling=(Cancelling) # {0} - file name @@ -105,7 +105,7 @@ DataResultViewerThumbnail.pageNextButton.text= DataResultViewerThumbnail.imagesLabel.text=Images: DataResultViewerThumbnail.imagesRangeLabel.text=- DataResultViewerThumbnail.pageNumLabel.text=- -DataResultViewerThumbnail.filePathLabel.text=\ +DataResultViewerThumbnail.filePathLabel.text=\ \ \ DataResultViewerThumbnail.goToPageLabel.text=Go to Page: DataResultViewerThumbnail.goToPageField.text= AdvancedConfigurationDialog.cancelButton.text=Cancel diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataArtifactContentViewer.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataArtifactContentViewer.java index a210cdfc36..04f3d56a35 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataArtifactContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataArtifactContentViewer.java @@ -385,7 +385,6 @@ public class DataArtifactContentViewer extends javax.swing.JPanel implements Dat case DATA_ARTIFACT: return MORE_PREFERRED; // everything else is less preferred - case ANALYSIS_RESULT: default: return LESS_PREFERRED; } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED index 18e279dd2c..a0d535f8e6 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/coreutils/Bundle.properties-MERGED @@ -30,7 +30,9 @@ PlatformUtil.getProcVmUsed.sigarNotInit.msg=Cannot get virt mem used, sigar not PlatformUtil.getProcVmUsed.gen.msg=Cannot get virt mem used, {0} PlatformUtil.getJvmMemInfo.usageText=JVM heap usage: {0}, JVM non-heap usage: {1} PlatformUtil.getPhysicalMemInfo.usageText=Physical memory usage (max, total, free): {0}, {1}, {2} -PlatformUtil.getAllMemUsageInfo.usageText={0}\n{1}\nProcess Virtual Memory: {2} +PlatformUtil.getAllMemUsageInfo.usageText={0}\n\ +{1}\n\ +Process Virtual Memory: {2} # {0} - file name ReadImageTask.mesageText=Reading image: {0} StringExtract.illegalStateException.cannotInit.msg=Unicode table not properly initialized, cannot instantiate StringExtract diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties index 034149f5ec..78fe3b05ee 100644 --- a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties @@ -12,7 +12,7 @@ DateSearchPanel.changedCheckBox.text=Changed DateSearchPanel.modifiedCheckBox.text=Modified DateSearchPanel.jLabel1.text=to NameSearchPanel.nameCheckBox.text=Name: -NameSearchPanel.noteNameLabel.text=*Note: Name match is case insensitive and matches any part of the file name. Regular expressions are not currently supported. +NameSearchPanel.noteNameLabel.text=*Note: Name match is case insensitive and matches any part of the file name (not including parent path). Regular expressions are not currently supported. NameSearchPanel.searchTextField.text= SizeSearchPanel.sizeCheckBox.text=Size: NameSearchPanel.cutMenuItem.text=Cut diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED index 4f0fdfbb56..075a0e7afb 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED @@ -1,7 +1,10 @@ DataSourceFilter.errorMessage.emptyDataSource=At least one data source must be selected. DateSearchFilter.errorMessage.endDateBeforeStartDate=The end date should be after the start date. DateSearchFilter.errorMessage.noCheckboxSelected=At least one date type checkbox must be selected. +FileSearchPanel.cancelledSearch.text=Search Was Cancelled FileSearchPanel.emptyNode.display.text=No results found. +FileSearchPanel.searchingNode.display.text=Performing file search by attributes. Please wait. +FileSearchPanel.searchingPath.text=File Search In Progress HashSearchFilter.errorMessage.emptyHash=Hash data is empty. HashSearchFilter.errorMessage.wrongCharacter=MD5 contains invalid hex characters. # {0} - hash data length @@ -16,7 +19,7 @@ KnownStatusSearchPanel.knownCheckBox.text=Known Status: KnownStatusSearchPanel.knownBadOptionCheckBox.text=Notable KnownStatusSearchPanel.knownOptionCheckBox.text=Known (NSRL or other) KnownStatusSearchPanel.unknownOptionCheckBox.text=Unknown -DateSearchFilter.noneSelectedMsg.text=At least one date type must be selected! +DateSearchFilter.noneSelectedMsg.text=At least one date type must be selected\! DateSearchPanel.dateCheckBox.text=Date: DateSearchPanel.jLabel4.text=Timezone: DateSearchPanel.createdCheckBox.text=Created @@ -25,7 +28,7 @@ DateSearchPanel.changedCheckBox.text=Changed DateSearchPanel.modifiedCheckBox.text=Modified DateSearchPanel.jLabel1.text=to NameSearchPanel.nameCheckBox.text=Name: -NameSearchPanel.noteNameLabel.text=*Note: Name match is case insensitive and matches any part of the file name. Regular expressions are not currently supported. +NameSearchPanel.noteNameLabel.text=*Note: Name match is case insensitive and matches any part of the file name (not including parent path). Regular expressions are not currently supported. NameSearchPanel.searchTextField.text= SearchNode.getName.text=Search Result SizeSearchFilter.errorMessage.nonNegativeNumber=Input size data is a negative number. @@ -57,7 +60,7 @@ FileSearchPanel.search.results.details=Large number of matches may impact perfor FileSearchPanel.search.exception.noFilterSelected.msg=At least one filter must be selected. FileSearchPanel.search.validationErr.msg=Validation Error: {0} FileSearchPanel.emptyWhereClause.text=Invalid options, nothing to show. -KnownStatusSearchFilter.noneSelectedMsg.text=At least one known status must be selected! +KnownStatusSearchFilter.noneSelectedMsg.text=At least one known status must be selected\! NameSearchFilter.emptyNameMsg.text=Must enter something for name search. SizeSearchPanel.sizeCompareComboBox.equalTo=equal to SizeSearchPanel.sizeCompareComboBox.greaterThan=greater than diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/FileSearchPanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/FileSearchPanel.java index 4801227416..62f6332c83 100644 --- a/Core/src/org/sleuthkit/autopsy/filesearch/FileSearchPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/FileSearchPanel.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,15 +29,19 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; import java.util.logging.Level; import javax.swing.JLabel; import javax.swing.JPanel; +import javax.swing.SwingWorker; import javax.swing.border.EmptyBorder; import org.openide.DialogDisplayer; import org.openide.NotifyDescriptor; import org.openide.nodes.Node; import org.openide.util.NbBundle; import org.openide.windows.TopComponent; +import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; @@ -56,9 +60,12 @@ import org.sleuthkit.datamodel.TskCoreException; @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives class FileSearchPanel extends javax.swing.JPanel { + private static final Logger logger = Logger.getLogger(FileSearchPanel.class.getName()); + private static final long serialVersionUID = 1L; private final List filters = new ArrayList<>(); private static int resultWindowCount = 0; //keep track of result windows so they get unique names private static final String EMPTY_WHERE_CLAUSE = NbBundle.getMessage(DateSearchFilter.class, "FileSearchPanel.emptyWhereClause.text"); + private static SwingWorker searchWorker = null; enum EVENT { CHECKED @@ -77,39 +84,39 @@ class FileSearchPanel extends javax.swing.JPanel { * This method is called from within the constructor to initialize the form. */ private void customizeComponents() { - + JLabel label = new JLabel(NbBundle.getMessage(this.getClass(), "FileSearchPanel.custComp.label.text")); label.setAlignmentX(Component.LEFT_ALIGNMENT); label.setBorder(new EmptyBorder(0, 0, 10, 0)); - + JPanel panel1 = new JPanel(); - panel1.setLayout(new GridLayout(1,2)); + panel1.setLayout(new GridLayout(1, 2)); panel1.add(new JLabel("")); JPanel panel2 = new JPanel(); - panel2.setLayout(new GridLayout(1,2, 20, 0)); + panel2.setLayout(new GridLayout(1, 2, 20, 0)); JPanel panel3 = new JPanel(); - panel3.setLayout(new GridLayout(1,2, 20, 0)); + panel3.setLayout(new GridLayout(1, 2, 20, 0)); JPanel panel4 = new JPanel(); - panel4.setLayout(new GridLayout(1,2, 20, 0)); + panel4.setLayout(new GridLayout(1, 2, 20, 0)); JPanel panel5 = new JPanel(); - panel5.setLayout(new GridLayout(1,2, 20, 0)); + panel5.setLayout(new GridLayout(1, 2, 20, 0)); // Create and add filter areas - NameSearchFilter nameFilter = new NameSearchFilter(); + NameSearchFilter nameFilter = new NameSearchFilter(); SizeSearchFilter sizeFilter = new SizeSearchFilter(); DateSearchFilter dateFilter = new DateSearchFilter(); KnownStatusSearchFilter knowStatusFilter = new KnownStatusSearchFilter(); HashSearchFilter hashFilter = new HashSearchFilter(); MimeTypeFilter mimeTypeFilter = new MimeTypeFilter(); DataSourceFilter dataSourceFilter = new DataSourceFilter(); - - panel2.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.name"),nameFilter)); - - panel3.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.metadata"),sizeFilter)); - - panel2.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.metadata"), dateFilter)); + + panel2.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.name"), nameFilter)); + + panel3.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.metadata"), sizeFilter)); + + panel2.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.metadata"), dateFilter)); panel3.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.knownStatus"), knowStatusFilter)); - + panel5.add(new FilterArea(NbBundle.getMessage(this.getClass(), "HashSearchPanel.md5CheckBox.text"), hashFilter)); panel5.add(new JLabel("")); panel4.add(new FilterArea(NbBundle.getMessage(this.getClass(), "FileSearchPanel.filterTitle.metadata"), mimeTypeFilter)); @@ -119,7 +126,7 @@ class FileSearchPanel extends javax.swing.JPanel { filterPanel.add(panel3); filterPanel.add(panel4); filterPanel.add(panel5); - + filters.add(nameFilter); filters.add(sizeFilter); filters.add(dateFilter); @@ -127,7 +134,7 @@ class FileSearchPanel extends javax.swing.JPanel { filters.add(hashFilter); filters.add(mimeTypeFilter); filters.add(dataSourceFilter); - + for (FileSearchFilter filter : this.getFilters()) { filter.addPropertyChangeListener(new PropertyChangeListener() { @Override @@ -141,7 +148,7 @@ class FileSearchPanel extends javax.swing.JPanel { public void actionPerformed(ActionEvent e) { search(); } - }); + }); searchButton.setEnabled(isValidSearch()); } @@ -168,56 +175,102 @@ class FileSearchPanel extends javax.swing.JPanel { * Action when the "Search" button is pressed. * */ - @NbBundle.Messages("FileSearchPanel.emptyNode.display.text=No results found.") + @NbBundle.Messages({"FileSearchPanel.emptyNode.display.text=No results found.", + "FileSearchPanel.searchingNode.display.text=Performing file search by attributes. Please wait.", + "FileSearchPanel.searchingPath.text=File Search In Progress", + "FileSearchPanel.cancelledSearch.text=Search Was Cancelled"}) private void search() { - // change the cursor to "waiting cursor" for this operation - this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + if (searchWorker != null && searchWorker.isDone()) { + searchWorker.cancel(true); + } try { if (this.isValidSearch()) { - String title = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.title", ++resultWindowCount); - String pathText = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.pathText"); - // try to get the number of matches first Case currentCase = Case.getCurrentCaseThrows(); // get the most updated case - long totalMatches = 0; - List contentList = null; - try { - SleuthkitCase tskDb = currentCase.getSleuthkitCase(); - contentList = tskDb.findAllFilesWhere(this.getQuery()); - - } catch (TskCoreException ex) { - Logger logger = Logger.getLogger(this.getClass().getName()); - logger.log(Level.WARNING, "Error while trying to get the number of matches.", ex); //NON-NLS - } - - if (contentList == null) { - contentList = Collections.emptyList(); - } - - SearchNode sn = new SearchNode(contentList); - TableFilterNode tableFilterNode = new TableFilterNode(sn, true, sn.getName()); - final TopComponent searchResultWin; - if (contentList.isEmpty()) { - Node emptyNode = new TableFilterNode(new EmptyNode(Bundle.FileSearchPanel_emptyNode_display_text()), true); - searchResultWin = DataResultTopComponent.createInstance(title, pathText, + Node emptyNode = new TableFilterNode(new EmptyNode(Bundle.FileSearchPanel_searchingNode_display_text()), true); + String title = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.title", ++resultWindowCount); + String pathText = Bundle.FileSearchPanel_searchingPath_text(); + final DataResultTopComponent searchResultWin = DataResultTopComponent.createInstance(title, pathText, emptyNode, 0); - } else { - searchResultWin = DataResultTopComponent.createInstance(title, pathText, - tableFilterNode, contentList.size()); - } searchResultWin.requestActive(); // make it the active top component + searchResultWin.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + searchWorker = new SwingWorker() { + List contentList = null; - /** - * If total matches more than 1000, pop up a dialog box that say - * the performance maybe be slow and to increase the - * performance, tell the users to refine their search. - */ - if (totalMatches > 10000) { - // show info - String msg = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.msg", totalMatches); - String details = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.details"); - MessageNotifyUtil.Notify.info(msg, details); + @Override + protected TableFilterNode doInBackground() throws Exception { + try { + SleuthkitCase tskDb = currentCase.getSleuthkitCase(); + contentList = tskDb.findAllFilesWhere(getQuery()); + + } catch (TskCoreException ex) { + Logger logger = Logger.getLogger(this.getClass().getName()); + logger.log(Level.WARNING, "Error while trying to get the number of matches.", ex); //NON-NLS + } + if (contentList == null) { + contentList = Collections.emptyList(); + } + if (contentList.isEmpty()) { + return new TableFilterNode(new EmptyNode(Bundle.FileSearchPanel_emptyNode_display_text()), true); + } + SearchNode sn = new SearchNode(contentList); + return new TableFilterNode(sn, true, sn.getName()); + } + + @Override + protected void done() { + String pathText = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.pathText"); + try { + TableFilterNode tableFilterNode = get(); + if (tableFilterNode == null) { //just incase this get() gets modified to return null or somehow can return null + tableFilterNode = new TableFilterNode(new EmptyNode(Bundle.FileSearchPanel_emptyNode_display_text()), true); + } + if (searchResultWin != null && searchResultWin.isOpened()) { + searchResultWin.setNode(tableFilterNode); + searchResultWin.requestActive(); // make it the active top component + } + /** + * If total matches more than 1000, pop up a dialog + * box that say the performance maybe be slow and to + * increase the performance, tell the users to + * refine their search. + */ + if (contentList.size() > 10000) { + // show info + String msg = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.msg", contentList.size()); + String details = NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.results.details"); + MessageNotifyUtil.Notify.info(msg, details); + } + } catch (InterruptedException | ExecutionException ex) { + logger.log(Level.SEVERE, "Error while performing file search by attributes", ex); + } catch (CancellationException ex) { + if (searchResultWin != null && searchResultWin.isOpened()) { + Node emptyNode = new TableFilterNode(new EmptyNode(Bundle.FileSearchPanel_cancelledSearch_text()), true); + searchResultWin.setNode(emptyNode); + pathText = Bundle.FileSearchPanel_cancelledSearch_text(); + } + logger.log(Level.INFO, "File search by attributes was cancelled", ex); + } finally { + if (searchResultWin != null && searchResultWin.isOpened()) { + searchResultWin.setPath(pathText); + searchResultWin.requestActive(); // make it the active top component + searchResultWin.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + } + } + }; + if (searchResultWin != null && searchResultWin.isOpened()) { + searchResultWin.addPropertyChangeListener(new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + if (evt.getPropertyName().equals("tcClosed") && !searchWorker.isDone() && evt.getOldValue() == null) { + searchWorker.cancel(true); + logger.log(Level.INFO, "User has closed the results window while search was in progress, search will be cancelled"); + } + } + }); } + searchWorker.execute(); } else { throw new FilterValidationException( NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.exception.noFilterSelected.msg")); @@ -226,8 +279,6 @@ class FileSearchPanel extends javax.swing.JPanel { NotifyDescriptor d = new NotifyDescriptor.Message( NbBundle.getMessage(this.getClass(), "FileSearchPanel.search.validationErr.msg", ex.getMessage())); DialogDisplayer.getDefault().notify(d); - } finally { - this.setCursor(null); } } diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java index 6fba326497..662cf7a067 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/MapWaypoint.java @@ -219,7 +219,7 @@ public final class MapWaypoint extends KdTree.XYZPoint implements org.jxmapviewe JMenuItem[] getMenuItems() throws TskCoreException { List menuItems = new ArrayList<>(); BlackboardArtifact artifact = dataModelWaypoint.getArtifact(); - Content content = artifact.getSleuthkitCase().getContentById(artifact.getObjectID()); + Content content = dataModelWaypoint.getContent(); menuItems.addAll(getTimelineMenuItems(dataModelWaypoint.getArtifact())); menuItems.addAll(getDataModelActionFactoryMenuItems(artifact, content)); diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java index 223b1e9cb3..13fda6f7e1 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java @@ -28,6 +28,7 @@ import java.util.Set; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; /** @@ -44,6 +45,7 @@ public class Waypoint { final private AbstractFile image; final private BlackboardArtifact artifact; final private GeoPath parentGeoPath; + final private Content content; final private List propertiesList; @@ -93,6 +95,11 @@ public class Waypoint { this.parentGeoPath = parentGeoPath; propertiesList = createGeolocationProperties(attributeMap); + try { + content = artifact.getSleuthkitCase().getContentById(artifact.getObjectID()); + } catch (TskCoreException ex) { + throw new GeoLocationDataException(String.format("Failed to get contend for artifact id (%d)", artifact.getId()), ex); + } } /** @@ -248,6 +255,10 @@ public class Waypoint { } return list; } + + public Content getContent() { + return content; + } /** * Simple property class for waypoint properties that a purely diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 4b5b431537..cda1ec1503 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -20,6 +20,7 @@ IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestJob.progress.dataSourceIngest.initialDisplayName=Analyzing {0} IngestJob.progress.dataSourceIngest.displayName={0} for {1} IngestJob.progress.fileIngest.displayName=Analyzing files from {0} +IngestJob.progress.dataArtifactIngest.displayName=Analyzing data artifacts from {0} IngestJob.progress.cancelling=Cancelling... IngestJob.cancellationDialog.title=Cancel Ingest IngestDialog.startButton.title=Start @@ -83,14 +84,15 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID IngestJobTableModel.colName.jobID=Job ID IngestJobTableModel.colName.dataSource=Data Source IngestJobTableModel.colName.start=Start -IngestJobTableModel.colName.numProcessed=Num Processed +IngestJobTableModel.colName.numProcessed=Files Processed IngestJobTableModel.colName.filesPerSec=Files/Sec IngestJobTableModel.colName.inProgress=In Progress IngestJobTableModel.colName.filesQueued=Files Queued -IngestJobTableModel.colName.dirQueued=Dir Queued -IngestJobTableModel.colName.rootQueued=Root Queued -IngestJobTableModel.colName.streamingQueued=Streaming Queued +IngestJobTableModel.colName.dirQueued=Dirs Queued +IngestJobTableModel.colName.rootQueued=Roots Queued +IngestJobTableModel.colName.streamingQueued=Streamed Files Queued IngestJobTableModel.colName.dsQueued=DS Queued +IngestJobTableModel.colName.artifactsQueued=Artifacts Queued ModuleTableModel.colName.module=Module ModuleTableModel.colName.duration=Duration IngestJobSettingsPanel.jButtonSelectAll.text=Select All diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED index 621713dfdf..9c1a1bfa5c 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties-MERGED @@ -36,6 +36,7 @@ IngestMessagePanel.totalUniqueMessagesNameVal.text=- IngestJob.progress.dataSourceIngest.initialDisplayName=Analyzing {0} IngestJob.progress.dataSourceIngest.displayName={0} for {1} IngestJob.progress.fileIngest.displayName=Analyzing files from {0} +IngestJob.progress.dataArtifactIngest.displayName=Analyzing data artifacts from {0} IngestJob.progress.cancelling=Cancelling... IngestJob.cancellationDialog.title=Cancel Ingest IngestDialog.startButton.title=Start @@ -99,14 +100,15 @@ IngestProgressSnapshotPanel.SnapshotsTableModel.colNames.jobID=Job ID IngestJobTableModel.colName.jobID=Job ID IngestJobTableModel.colName.dataSource=Data Source IngestJobTableModel.colName.start=Start -IngestJobTableModel.colName.numProcessed=Num Processed +IngestJobTableModel.colName.numProcessed=Files Processed IngestJobTableModel.colName.filesPerSec=Files/Sec IngestJobTableModel.colName.inProgress=In Progress IngestJobTableModel.colName.filesQueued=Files Queued -IngestJobTableModel.colName.dirQueued=Dir Queued -IngestJobTableModel.colName.rootQueued=Root Queued -IngestJobTableModel.colName.streamingQueued=Streaming Queued +IngestJobTableModel.colName.dirQueued=Dirs Queued +IngestJobTableModel.colName.rootQueued=Roots Queued +IngestJobTableModel.colName.streamingQueued=Streamed Files Queued IngestJobTableModel.colName.dsQueued=DS Queued +IngestJobTableModel.colName.artifactsQueued=Artifacts Queued ModuleTableModel.colName.module=Module ModuleTableModel.colName.duration=Duration IngestJobSettingsPanel.jButtonSelectAll.text=Select All @@ -142,7 +144,7 @@ IngestJob.cancelReason.outOfDiskSpace.text=Out of disk space IngestJob.cancelReason.servicesDown.text=Services Down IngestJob.cancelReason.caseClosed.text=Case closed IngestJobSettingsPanel.globalSettingsButton.text=Global Settings -gest= +gest IngestJobSettingsPanel.globalSettingsButton.actionCommand=Advanced IngestJobSettingsPanel.globalSettingsButton.text=Global Settings IngestJobSettingsPanel.pastJobsButton.text=History diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java new file mode 100755 index 0000000000..a37816d6be --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestModule.java @@ -0,0 +1,46 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.DataArtifact; + +/** + * Interface that must be implemented by all ingest modules that process data + * artifacts. + */ +public interface DataArtifactIngestModule extends IngestModule { + + /** + * Processes a data artifact. + * + * IMPORTANT: In addition to returning ProcessResult.OK or + * ProcessResult.ERROR, modules should log all errors using methods provided + * by the org.sleuthkit.autopsy.coreutils.Logger class. Log messages should + * include the name and object ID of the data being processed. If an + * exception has been caught by the module, the exception should be sent to + * the Logger along with the log message so that a stack trace will appear + * in the application log. + * + * @param artifact The artifact to process. + * + * @return A result code indicating success or failure of the processing. + */ + ProcessResult process(DataArtifact artifact); + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java new file mode 100755 index 0000000000..824d7d7fe9 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestPipeline.java @@ -0,0 +1,90 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import java.util.List; +import java.util.Optional; +import org.sleuthkit.datamodel.DataArtifact; + +/** + * A pipeline of data artifact ingest modules used to execute data artifact + * ingest tasks for an ingest job. + */ +final class DataArtifactIngestPipeline extends IngestTaskPipeline { + + /** + * Constructs a pipeline of data artifact ingest modules used to execute + * data artifact ingest tasks for an ingest job. + * + * @param ingestJobPipeline The ingest job pipeline that owns this ingest + * task pipeline. + * @param moduleTemplates The ingest module templates that define this + * pipeline. May be an empty list. + */ + DataArtifactIngestPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { + super(ingestJobPipeline, moduleTemplates); + } + + @Override + Optional> acceptModuleTemplate(IngestModuleTemplate template) { + Optional> module = Optional.empty(); + if (template.isDataArtifactIngestModuleTemplate()) { + DataArtifactIngestModule ingestModule = template.createDataArtifactIngestModule(); + module = Optional.of(new DataArtifactIngestPipelineModule(ingestModule, template.getModuleName())); + } + return module; + } + + @Override + void prepareForTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { + } + + @Override + void cleanUpAfterTask(DataArtifactIngestTask task) throws IngestTaskPipelineException { + } + + /** + * A decorator that adds ingest infrastructure operations to a data artifact + * ingest module. + */ + static final class DataArtifactIngestPipelineModule extends IngestTaskPipeline.PipelineModule { + + private final DataArtifactIngestModule module; + + /** + * Constructs a decorator that adds ingest infrastructure operations to + * a data artifact ingest module. + * + * @param module The module. + * @param displayName The display name of the module. + */ + DataArtifactIngestPipelineModule(DataArtifactIngestModule module, String displayName) { + super(module, displayName); + this.module = module; + } + + @Override + void executeTask(IngestJobPipeline ingestJobPipeline, DataArtifactIngestTask task) throws IngestModuleException { + DataArtifact artifact = task.getDataArtifact(); + module.process(artifact); + } + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java new file mode 100755 index 0000000000..d71318236e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataArtifactIngestTask.java @@ -0,0 +1,59 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.datamodel.DataArtifact; + +/** + * A data artifact ingest task that will be executed by an ingest thread using a + * given ingest job pipeline. + */ +final class DataArtifactIngestTask extends IngestTask { + + private final DataArtifact artifact; + + /** + * Constructs a data artifact ingest task that will be executed by an ingest + * thread using a given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline to use to execute the + * task. + * @param artifact The data artifact to be processed. + */ + DataArtifactIngestTask(IngestJobPipeline ingestJobPipeline, DataArtifact artifact) { + super(ingestJobPipeline); + this.artifact = artifact; + } + + /** + * Gets the data artifact for this task. + * + * @return The data artifact. + */ + DataArtifact getDataArtifact() { + return artifact; + } + + @Override + void execute(long threadId) { + super.setThreadId(threadId); + getIngestJobPipeline().execute(this); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java index 4411594995..c76f7a9b43 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,10 +30,19 @@ public interface DataSourceIngestModule extends IngestModule { * Processes a data source. Called once between calls to startUp() and * shutDown(). * + * IMPORTANT: In addition to returning ProcessResult.OK or + * ProcessResult.ERROR, modules should log all errors using methods provided + * by the org.sleuthkit.autopsy.coreutils.Logger class. Log messages should + * include the name and object ID of the data being processed. If an + * exception has been caught by the module, the exception should be sent to + * the Logger along with the log message so that a stack trace will appear + * in the application log. + * * @param dataSource The data source to process. * @param progressBar A progress bar to be used to report progress. * * @return A result code indicating success or failure of the processing. */ ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar); + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java index 83b5a1457f..f4f58e1503 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleAdapter.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,20 +21,15 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.Content; /** + * DO NOT USE: As of Java 8, interfaces can have default methods. IngestModule + * now provides default no-op versions of startUp() and shutDown(). This class + * is no longer needed and can be DEPRECATED when convenient. + * * An adapter that provides a no-op implementation of the startUp() method for * data source ingest modules. - * - * NOTE: As of Java 8, interfaces can have default methods. - * DataSourceIngestModule now provides default no-op versions of startUp() and - * shutDown(). This class is no longer needed and can be deprecated when - * convenient. */ public abstract class DataSourceIngestModuleAdapter implements DataSourceIngestModule { - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - } - @Override public abstract ProcessResult process(Content dataSource, DataSourceIngestModuleProgress progressBar); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index f48fc337f0..1feb13545b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -26,7 +26,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; /** - * A pipeline of data source level ingest modules for performing data source + * A pipeline of data source level ingest modules for executing data source * level ingest tasks for an ingest job. */ final class DataSourceIngestPipeline extends IngestTaskPipeline { @@ -57,11 +57,11 @@ final class DataSourceIngestPipeline extends IngestTaskPipeline sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,15 +18,27 @@ */ package org.sleuthkit.autopsy.ingest; +/** + * A data source level ingest task that will be executed by an ingest thread + * using a given ingest job pipeline. + */ final class DataSourceIngestTask extends IngestTask { + /** + * Constructs a data source level ingest task that will be executed by an + * ingest thread using a given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline to use to execute the + * task. + */ DataSourceIngestTask(IngestJobPipeline ingestJobPipeline) { super(ingestJobPipeline); } @Override - void execute(long threadId) throws InterruptedException { + void execute(long threadId) { super.setThreadId(threadId); - getIngestJobPipeline().process(this); - } + getIngestJobPipeline().execute(this); + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java index 60355e10f8..f13a614e54 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModule.java @@ -30,6 +30,14 @@ public interface FileIngestModule extends IngestModule { * Processes a file. Called between calls to startUp() and shutDown(). Will * be called for each file in a data source. * + * IMPORTANT: In addition to returning ProcessResult.OK or + * ProcessResult.ERROR, modules should log all errors using methods provided + * by the org.sleuthkit.autopsy.coreutils.Logger class. Log messages should + * include the name and object ID of the data being processed. If an + * exception has been caught by the module, the exception should be sent to + * the Logger along with the log message so that a stack trace will appear + * in the application log. + * * @param file The file to analyze. * * @return A result code indicating success or failure of the processing. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java index 9f733308c8..ea553013f2 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestModuleAdapter.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,23 +21,17 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.AbstractFile; /** + * DO NOT USE: As of Java 8, interfaces can have default methods. IngestModule + * now provides default no-op versions of startUp() and shutDown(). This class + * is no longer needed and can be DEPRECATED when convenient. + * * * An adapter that provides no-op implementations of the startUp() and * shutDown() methods for file ingest modules. * - * NOTE: As of Java 8, interfaces can have default methods. FileIngestModule now - * provides default no-op versions of startUp() and shutDown(). This class is no - * longer needed and can be deprecated when convenient. */ public abstract class FileIngestModuleAdapter implements FileIngestModule { - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - } - @Override public abstract ProcessResult process(AbstractFile file); - @Override - public void shutDown() { - } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index f7f8a8f555..295a463db8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -33,7 +33,7 @@ import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; import org.sleuthkit.datamodel.TskCoreException; /** - * A pipeline of file ingest modules for performing file ingest tasks for an + * A pipeline of file ingest modules for executing file ingest tasks for an * ingest job. */ @NbBundle.Messages({ @@ -49,7 +49,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { private final List fileBatch; /** - * Constructs a pipeline of file ingest modules for performing file ingest + * Constructs a pipeline of file ingest modules for executing file ingest * tasks for an ingest job. * * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. @@ -73,11 +73,11 @@ final class FileIngestPipeline extends IngestTaskPipeline { } @Override - void prepareTask(FileIngestTask task) throws IngestTaskPipelineException { + void prepareForTask(FileIngestTask task) throws IngestTaskPipelineException { } @Override - void completeTask(FileIngestTask task) throws IngestTaskPipelineException { + void cleanUpAfterTask(FileIngestTask task) throws IngestTaskPipelineException { try { ingestManager.setIngestTaskProgress(task, SAVE_RESULTS_ACTIVITY); AbstractFile file = task.getFile(); @@ -106,7 +106,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { } /** - * Adds a file to a file cache used to update the case database with new + * Adds a file to a file cache used to update the case database with any new * properties added to the files in the cache by the ingest modules that * processed them. If adding the file to the cache fills the cache, a batch * update is done immediately. @@ -195,7 +195,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { } @Override - void performTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { + void executeTask(IngestJobPipeline ingestJobPipeline, FileIngestTask task) throws IngestModuleException { AbstractFile file = null; try { file = task.getFile(); @@ -203,12 +203,7 @@ final class FileIngestPipeline extends IngestTaskPipeline { throw new IngestModuleException(String.format("Failed to get file (file objId = %d)", task.getFileId()), ex); //NON-NLS } ingestManager.setIngestTaskProgress(task, getDisplayName()); - ingestJobPipeline.setCurrentFileIngestModule(getDisplayName(), file.getName()); - ProcessResult result = module.process(file); - // See JIRA-7449 -// if (result == ProcessResult.ERROR) { -// throw new IngestModuleException(String.format("%s experienced an error analyzing %s (file objId = %d)", getDisplayName(), file.getName(), file.getId())); //NON-NLS -// } + module.process(file); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index 8bb918cce1..280c81c57b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2012-2015 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,29 +24,60 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * Represents a single file analysis task, which is defined by a file to analyze - * and the InjestJob/Pipeline to run it on. + * A file ingest task that will be executed by an ingest thread using a given + * ingest job pipeline. */ final class FileIngestTask extends IngestTask { - - private final long fileId; - private AbstractFile file = null; + private final long fileId; + private AbstractFile file; + + /** + * Constructs a file ingest task that will be executed by an ingest thread + * using a given ingest job pipeline. + * + * @param ingestJobPipeline The ingest job pipeline to use to execute the + * task. + * @param file The file to be processed. + */ FileIngestTask(IngestJobPipeline ingestJobPipeline, AbstractFile file) { super(ingestJobPipeline); this.file = file; fileId = file.getId(); } + /** + * Constructs a file ingest task that will be executed by an ingest thread + * using a given ingest job pipeline. This constructor supports streaming + * ingest by deferring the construction of the AbstractFile object for this + * task to conserve heap memory. + * + * @param ingestJobPipeline The ingest job pipeline to use to execute the + * task. + * @param fileId The object ID of the file to be processed. + */ FileIngestTask(IngestJobPipeline ingestJobPipeline, long fileId) { super(ingestJobPipeline); this.fileId = fileId; } + /** + * Gets the object ID of the file for this task. + * + * @return The object ID. + */ long getFileId() { return fileId; } + /** + * Gets the file for this task. + * + * @return The file. + * + * @throws TskCoreException The exception is thrown if there is an error + * retieving the file from the case database. + */ synchronized AbstractFile getFile() throws TskCoreException { if (file == null) { file = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(fileId); @@ -55,11 +86,11 @@ final class FileIngestTask extends IngestTask { } @Override - void execute(long threadId) throws InterruptedException { + void execute(long threadId) { super.setThreadId(threadId); - getIngestJobPipeline().process(this); - } - + getIngestJobPipeline().execute(this); + } + @Override public boolean equals(Object obj) { if (obj == null) { @@ -84,4 +115,5 @@ final class FileIngestTask extends IngestTask { hash = 47 * hash + Objects.hashCode(this.fileId); return hash; } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 76bbec2365..321d38c627 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -33,7 +33,6 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DataSource; /** * Analyzes one or more data sources using a set of ingest modules specified via @@ -120,7 +119,7 @@ public final class IngestJob { * * @param settings The ingest job settings. */ - IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) { + IngestJob(Content dataSource, Mode ingestMode, IngestJobSettings settings) { this.id = IngestJob.nextId.getAndIncrement(); this.ingestJobPipelines = new ConcurrentHashMap<>(); this.dataSources.add(dataSource); @@ -138,7 +137,7 @@ public final class IngestJob { public long getId() { return this.id; } - + /** * Checks to see if this ingest job has at least one non-empty ingest module * pipeline (first or second stage data-source-level pipeline or file-level @@ -162,7 +161,7 @@ public final class IngestJob { } // Streaming ingest jobs will only have one data source IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.addStreamingIngestFiles(fileObjIds); + streamingIngestPipeline.addStreamedFiles(fileObjIds); } /** @@ -175,7 +174,7 @@ public final class IngestJob { } // Streaming ingest jobs will only have one data source IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next(); - streamingIngestPipeline.processStreamingIngestDataSource(); + streamingIngestPipeline.addStreamedDataSource(); } /** @@ -184,33 +183,28 @@ public final class IngestJob { * * @return A collection of ingest module start up errors, empty on success. */ - List start() { - + List start() throws InterruptedException { /* - * Set up the pipeline(s) + * Set up the ingest job pipelines, one for each data source to be + * ingested by this job. */ if (files.isEmpty()) { for (Content dataSource : dataSources) { IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSource, settings); - this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); + ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } } else { IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSources.get(0), files, settings); - this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); + ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline); } incompleteJobsCount.set(ingestJobPipelines.size()); /* - * Try to start each data source ingest job. Note that there is an - * assumption here that if there is going to be a module startup - * failure, it will be for the first ingest job pipeline. - * - * TODO (RC): Consider separating module start up from pipeline startup - * so that no processing is done if this assumption is false. + * Try to start up each ingest job pipeline. Stop at the first failure. */ List errors = new ArrayList<>(); - for (IngestJobPipeline ingestJobPipeline : this.ingestJobPipelines.values()) { - errors.addAll(ingestJobPipeline.start()); + for (IngestJobPipeline ingestJobPipeline : ingestJobPipelines.values()) { + errors.addAll(ingestJobPipeline.startUp()); if (errors.isEmpty() == false) { break; } @@ -220,8 +214,8 @@ public final class IngestJob { * Handle start up success or failure. */ if (errors.isEmpty()) { - for (IngestJobPipeline dataSourceJob : this.ingestJobPipelines.values()) { - IngestManager.getInstance().fireDataSourceAnalysisStarted(id, dataSourceJob.getId(), dataSourceJob.getDataSource()); + for (IngestJobPipeline ingestJobPipeline : ingestJobPipelines.values()) { + IngestManager.getInstance().fireDataSourceAnalysisStarted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); } } else { cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED); @@ -337,7 +331,7 @@ public final class IngestJob { * * @param ingestJobPipeline A completed ingestJobPipeline. */ - void ingestJobPipelineFinished(IngestJobPipeline ingestJobPipeline) { + void notifyIngestPipelineShutDown(IngestJobPipeline ingestJobPipeline) { IngestManager ingestManager = IngestManager.getInstance(); if (!ingestJobPipeline.isCancelled()) { ingestManager.fireDataSourceAnalysisCompleted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource()); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 697e512ad8..6f1724d0b7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.List; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataArtifact; /** * Provides an ingest module with services specific to the ingest job of which @@ -30,81 +31,108 @@ public final class IngestJobContext { private final IngestJobPipeline ingestJobPipeline; + /** + * Constructs an ingest job context object that provides an ingest module + * with services specific to the ingest job of which the module is a part. + * + * @param ingestJobPipeline The ingest pipeline for the job. + */ IngestJobContext(IngestJobPipeline ingestJobPipeline) { this.ingestJobPipeline = ingestJobPipeline; } /** - * Gets the ingest job execution context identifier. + * Gets the execution context identifier of the ingest job. * * @return The context string. */ public String getExecutionContext() { - return this.ingestJobPipeline.getExecutionContext(); + return ingestJobPipeline.getExecutionContext(); } - + /** - * Gets the data source associated with this context. + * Gets the data source for the ingest job. * * @return The data source. */ public Content getDataSource() { - return this.ingestJobPipeline.getDataSource(); + return ingestJobPipeline.getDataSource(); } /** - * Gets the identifier of the ingest job associated with this context. + * Gets the unique identifier for the ingest job. * - * @return The ingest job identifier. + * @return The ID. */ public long getJobId() { - return this.ingestJobPipeline.getId(); + return ingestJobPipeline.getId(); } /** - * Queries whether or not cancellation of the data source ingest part of the - * ingest job associated with this context has been requested. + * Indicates whether or not cancellation of the ingest job has been + * requested. * * @return True or false. * - * @deprecated Use dataSourceIngestIsCancelled() or fileIngestIsCancelled() + * @deprecated Modules should call a type-specific cancellation check method * instead. */ @Deprecated public boolean isJobCancelled() { - return this.dataSourceIngestIsCancelled(); + return ingestJobPipeline.isCancelled(); } /** - * Allows a data source ingest module to determine whether or not - * cancellation of the data source ingest part of the ingest job associated - * with this context has been requested. + * Indicates whether or not cancellation of the currently running data + * source level ingest module has been requested. Data source level ingest + * modules should check this periodically and break off processing if the + * method returns true. * * @return True or false. */ public boolean dataSourceIngestIsCancelled() { - return this.ingestJobPipeline.currentDataSourceIngestModuleIsCancelled() || this.ingestJobPipeline.isCancelled(); + return ingestJobPipeline.currentDataSourceIngestModuleIsCancelled() || ingestJobPipeline.isCancelled(); } /** - * Allows a file ingest module to determine whether or not cancellation of - * the file ingest part of the ingest job associated with this context has - * been requested. + * Indicates whether or not cancellation of the currently running file level + * ingest module has been requested. File level ingest modules should check + * this periodically and break off processing if the method returns true. * * @return True or false. */ public boolean fileIngestIsCancelled() { - return this.ingestJobPipeline.isCancelled(); + /* + * It is not currently possible to cancel individual file ingest + * modules. + */ + return ingestJobPipeline.isCancelled(); + } + + /** + * Checks whether or not cancellation of the currently running data artifact + * ingest module for the ingest job has been requested. Data artifact ingest + * modules should check this periodically and break off processing if the + * method returns true. + * + * @return True or false. + */ + public boolean dataArtifactIngestIsCancelled() { + /* + * It is not currently possible to cancel individual data artifact + * ingest modules. + */ + return ingestJobPipeline.isCancelled(); } /** * Queries whether or not unallocated space should be processed for the - * ingest job associated with this context. + * ingest job. * * @return True or false. */ public boolean processingUnallocatedSpace() { - return this.ingestJobPipeline.shouldProcessUnallocatedSpace(); + return ingestJobPipeline.shouldProcessUnallocatedSpace(); } /** @@ -113,21 +141,31 @@ public final class IngestJobContext { * * @param files The files to be added. * - * @deprecated use addFilesToJob() instead + * @deprecated use addFilesToJob() instead. */ @Deprecated public void scheduleFiles(List files) { - this.addFilesToJob(files); + addFilesToJob(files); + } + + /** + * Adds one or more files, e.g., extracted or carved files, to the ingest + * job for processing by its file ingest modules. + * + * @param files The files. + */ + public void addFilesToJob(List files) { + ingestJobPipeline.addFiles(files); } /** - * Adds one or more files, i.e., extracted or carved files, to the ingest - * job associated with this context. + * Adds one or more data artifacts to the ingest job for processing by its + * data artifact ingest modules. * - * @param files The files to be added. + * @param artifacts The artifacts. */ - public void addFilesToJob(List files) { - this.ingestJobPipeline.addFiles(files); + public void addDataArtifactsToJob(List artifacts) { + ingestJobPipeline.addDataArtifacts(artifacts); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 392789a33e..c68dde5911 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -40,7 +40,6 @@ import org.openide.util.Cancellable; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; @@ -55,139 +54,164 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.modules.interestingitems.FilesSet; import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer; +import org.sleuthkit.datamodel.DataArtifact; +import org.sleuthkit.datamodel.DataSource; /** - * Encapsulates a data source and the ingest module pipelines used to process - * it. + * A pipeline of ingest modules for analyzing one of the data sources in an + * ingest job. The ingest modules are organized into child pipelines by ingest + * module type and are run in stages. */ final class IngestJobPipeline { - private static String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; + private static final String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; private static final Logger logger = Logger.getLogger(IngestJobPipeline.class.getName()); - // to match something like: "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" - private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); + /* + * A regular expression for identifying the proxy classes Jython generates + * for ingest module factories written using Python. For example: + * org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14 + */ + private static final Pattern JYTHON_MODULE_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); - /** - * These fields define a data source ingest job: the parent ingest job, an + /* + * These fields define an ingest pipeline: the parent ingest job, a pipeline * ID, the user's ingest job settings, and the data source to be analyzed. * Optionally, there is a set of files to be analyzed instead of analyzing * all of the files in the data source. + * + * The pipeline ID is used to associate the pipeline with the ingest tasks + * that the ingest task scheduler creates for the ingest job. The ingest job + * ID cannot be used for this purpose because the parent ingest job may have + * more than one data source and each data source gets its own pipeline. */ private final IngestJob parentJob; - private static final AtomicLong nextJobId = new AtomicLong(0L); - private final long id; + private static final AtomicLong nextPipelineId = new AtomicLong(0L); + private final long pipelineId; private final IngestJobSettings settings; - private Content dataSource = null; - private final IngestJob.Mode ingestMode; - private final List files = new ArrayList<>(); + private DataSource dataSource; + private final List files; - /** - * A data source ingest job runs in stages. + /* + * An ingest pipeline runs its ingest modules in stages. */ private static enum Stages { - - /** - * Setting up for processing. + /* + * The pipeline is instantiating ingest modules and loading them into + * its child ingest module pipelines. */ INITIALIZATION, - /** - * Running only file ingest modules (used only for streaming ingest) + /* + * This stage is unique to a streaming mode ingest job. The pipeline is + * running file ingest modules on files streamed to it via + * addStreamedFiles(). If configured to have data artifact ingest + * modules, the pipeline is also running them on data artifacts + * generated by the analysis of the streamed files. This stage ends when + * the data source is streamed to the pipeline via + * addStreamedDataSource(). */ - FIRST_STAGE_FILES_ONLY, - /** - * Running high priority data source level ingest modules and file level - * ingest modules. + FIRST_STAGE_STREAMING, + /* + * The pipeline is running the following three types of ingest modules: + * higher priority data source level ingest modules, file ingest + * modules, and data artifact ingest modules. */ - FIRST_STAGE_FILES_AND_DATASOURCE, + FIRST_STAGE, /** - * Running lower priority, usually long-running, data source level - * ingest modules. + * The pipeline is running lower priority, usually long-running, data + * source level ingest modules and data artifact ingest modules. */ SECOND_STAGE, /** - * Cleaning up. + * The pipeline is shutting down its ingest modules. */ FINALIZATION }; private volatile Stages stage = IngestJobPipeline.Stages.INITIALIZATION; - private final Object stageCompletionCheckLock = new Object(); - /** - * A data source ingest job has separate data source level ingest module - * pipelines for the first and second processing stages. Longer running, - * lower priority modules belong in the second stage pipeline, although this - * cannot be enforced. Note that the pipelines for both stages are created - * at job start up to allow for verification that they both can be started - * up without errors. + /* + * The stage field is volatile to allow it to be read by multiple threads. + * This lock is used not to guard the stage field, but to make stage + * transitions atomic. + */ + private final Object stageTransitionLock = new Object(); + + /* + * An ingest pipeline has separate data source level ingest module pipelines + * for the first and second stages. Longer running, lower priority modules + * belong in the second stage pipeline. */ - private final Object dataSourceIngestPipelineLock = new Object(); private DataSourceIngestPipeline firstStageDataSourceIngestPipeline; private DataSourceIngestPipeline secondStageDataSourceIngestPipeline; - private DataSourceIngestPipeline currentDataSourceIngestPipeline; + private volatile DataSourceIngestPipeline currentDataSourceIngestPipeline; - /** - * A data source ingest job has a collection of identical file level ingest - * module pipelines, one for each file level ingest thread in the ingest - * manager. A blocking queue is used to dole out the pipelines to the - * threads and an ordinary list is used when the ingest job needs to access - * the pipelines to query their status. + /* + * An ingest pipeline has a collection of identical file ingest module + * pipelines, one for each file ingest thread in the ingest manager. The + * file ingest threads take and return file ingest pipeline copies from a + * blocking queue as they work through the file ingest tasks for the ingest + * job. Additionally, a fixed list of all of the file ingest module + * pipelines is used to bypass the blocking queue when cycling through the + * pipelines to make ingest progress snapshots. */ private final LinkedBlockingQueue fileIngestPipelinesQueue = new LinkedBlockingQueue<>(); private final List fileIngestPipelines = new ArrayList<>(); - /** - * A data source ingest job supports cancellation of either the currently - * running data source level ingest module or the entire ingest job. - * - * TODO: The currentDataSourceIngestModuleCancelled field and all of the - * code concerned with it is a hack to avoid an API change. The next time an - * API change is legal, a cancel() method needs to be added to the - * IngestModule interface and this field should be removed. The "ingest job - * is canceled" queries should also be removed from the IngestJobContext - * class. + /* + * An ingest pipeline has a single data artifact ingest module pipeline. + */ + private DataArtifactIngestPipeline artifactIngestPipeline; + + /* + * An ingest pipeline supports cancellation of analysis by individual data + * source level ingest modules or cancellation of all remaining analysis by + * all of its ingest modules. Cancellation works by setting flags that are + * checked by the ingest module pipelines every time they transition from + * one module to another. Ingest modules are also expected to check these + * flags (via the ingest job context) and stop processing if they are set. + * This approach to cancellation means that there can be a variable length + * delay between a cancellation request and its fulfillment. Analysis + * already completed at the time that cancellation occurs is not discarded. */ private volatile boolean currentDataSourceIngestModuleCancelled; private final List cancelledDataSourceIngestModules = new CopyOnWriteArrayList<>(); private volatile boolean cancelled; private volatile IngestJob.CancellationReason cancellationReason = IngestJob.CancellationReason.NOT_CANCELLED; - /** - * A data source ingest job uses the task scheduler singleton to create and - * queue the ingest tasks that make up the job. + /* + * An ingest pipeline interacts with the ingest task scheduler to create + * ingest tasks for analyzing the data source, files and data artifacts that + * are the subject of the ingest job. The scheduler queues the tasks for the + * ingest manager's ingest threads. The ingest tasks are the units of work + * for the ingest pipeline's child ingest module pipelines. */ private static final IngestTasksScheduler taskScheduler = IngestTasksScheduler.getInstance(); - /** - * A data source ingest job can run interactively using NetBeans progress - * handles. + /* + * If running with a GUI, an ingest pipeline reports analysis progress and + * allows a user to cancel all or part of the analysis using progress bars + * in the lower right hand corner of the main application window. */ private final boolean doUI; - - /** - * A data source ingest job uses these fields to report data source level - * ingest progress. - */ private final Object dataSourceIngestProgressLock = new Object(); - private ProgressHandle dataSourceIngestProgress; - - /** - * A data source ingest job uses these fields to report file level ingest - * progress. - */ + private ProgressHandle dataSourceIngestProgressBar; private final Object fileIngestProgressLock = new Object(); private final List filesInProgress = new ArrayList<>(); private long estimatedFilesToProcess; private long processedFiles; - private ProgressHandle fileIngestProgress; - private String currentFileIngestModule = ""; - private String currentFileIngestTask = ""; - private final List ingestModules = new ArrayList<>(); - private volatile IngestJobInfo ingestJob; + private ProgressHandle fileIngestProgressBar; + private final Object artifactIngestProgressLock = new Object(); + private ProgressHandle artifactIngestProgressBar; + + /* + * Ingest job details are tracked using this object and are recorded in the + * case database when the pipeline starts up and shuts down. + */ + private volatile IngestJobInfo ingestJobInfo; /** - * A data source ingest job uses this field to report its creation time. + * An ingest pipeline uses this field to report its creation time. */ private final long createTime; @@ -202,103 +226,110 @@ final class IngestJobPipeline { private final Set pausedIngestThreads = new HashSet<>(); /** - * Constructs an object that encapsulates a data source and the ingest - * module pipelines used to analyze it. + * Constructs a pipeline of ingest modules for analyzing one of the data + * sources in an ingest job. The ingest modules are organized into child + * pipelines by ingest module type and are run in stages. * - * @param parentJob The ingest job of which this data source ingest job is - * a part. - * @param dataSource The data source to be ingested. - * @param settings The settings for the ingest job. + * @param parentJob The ingest job. + * @param dataSource The data source. + * @param settings The ingest job settings. + * + * @throws InterruptedException Exception thrown if the thread in which the + * pipeline is being created is interrupted. */ - IngestJobPipeline(IngestJob parentJob, Content dataSource, IngestJobSettings settings) { + IngestJobPipeline(IngestJob parentJob, Content dataSource, IngestJobSettings settings) throws InterruptedException { this(parentJob, dataSource, Collections.emptyList(), settings); } /** - * Constructs an object that encapsulates a data source and the ingest - * module pipelines used to analyze it. Either all of the files in the data - * source or a given subset of the files will be analyzed. + * Constructs a pipeline of ingest modules for analyzing one of the data + * sources in an ingest job. The ingest modules are organized into child + * pipelines by ingest module type and are run in stages. * - * @param parentJob The ingest job of which this data source ingest job is - * a part. - * @param dataSource The data source to be ingested. - * @param files A subset of the files for the data source. - * @param settings The settings for the ingest job. + * @param parentJob The ingest job. + * @param dataSource The data source. + * @param files A subset of the files from the data source. If the list + * is empty, ALL of the files in the data source are an + * analyzed. + * @param settings The ingest job settings. + * + * @throws InterruptedException Exception thrown if the thread in which the + * pipeline is being created is interrupted. */ - IngestJobPipeline(IngestJob parentJob, Content dataSource, List files, IngestJobSettings settings) { + IngestJobPipeline(IngestJob parentJob, Content dataSource, List files, IngestJobSettings settings) throws InterruptedException { + if (!(dataSource instanceof DataSource)) { + throw new IllegalArgumentException("Passed dataSource that does not implement the DataSource interface"); //NON-NLS + } this.parentJob = parentJob; - this.id = IngestJobPipeline.nextJobId.getAndIncrement(); - this.dataSource = dataSource; + pipelineId = IngestJobPipeline.nextPipelineId.getAndIncrement(); + this.dataSource = (DataSource) dataSource; + this.files = new ArrayList<>(); this.files.addAll(files); - this.ingestMode = parentJob.getIngestMode(); this.settings = settings; - this.doUI = RuntimeProperties.runningWithGUI(); - this.createTime = new Date().getTime(); - this.stage = Stages.INITIALIZATION; - this.createIngestPipelines(); + doUI = RuntimeProperties.runningWithGUI(); + createTime = new Date().getTime(); + stage = Stages.INITIALIZATION; + createIngestModulePipelines(); } /** - * Adds ingest modules to a list with autopsy modules first and third party - * modules next. + * Sorts ingest module templates so that core Autopsy ingest modules come + * before third party ingest modules and ingest modules implemented using + * Java come before ingest modules implemented using Jython. * - * @param dest The destination for the modules to be added. - * @param src A map of fully qualified class name mapped to the - * IngestModuleTemplate. - * @param jythonSrc A map of fully qualified class name mapped to the - * IngestModuleTemplate for jython modules. + * @param sortedModules The output list to hold the sorted modules. + * @param javaModules The input ingest module templates for modules + * implemented using Java. + * @param jythonModules The ingest module templates for modules implemented + * using Jython. */ - private static void addOrdered(final List dest, - final Map src, final Map jythonSrc) { - + private static void addToIngestPipelineTemplate(final List sortedModules, final Map javaModules, final Map jythonModules) { final List autopsyModules = new ArrayList<>(); final List thirdPartyModules = new ArrayList<>(); - - Stream.concat(src.entrySet().stream(), jythonSrc.entrySet().stream()).forEach((templateEntry) -> { + Stream.concat(javaModules.entrySet().stream(), jythonModules.entrySet().stream()).forEach((templateEntry) -> { if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) { autopsyModules.add(templateEntry.getValue()); } else { thirdPartyModules.add(templateEntry.getValue()); } }); - - dest.addAll(autopsyModules); - dest.addAll(thirdPartyModules); + sortedModules.addAll(autopsyModules); + sortedModules.addAll(thirdPartyModules); } /** - * Takes a classname like + * Extracts a module class name from a Jython module proxy class name. For + * example, a Jython class name such * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" - * and provides "GPX_Parser_Module.GPXParserFileIngestModuleFactory" or null - * if not in jython package. + * will be parsed to return + * "GPX_Parser_Module.GPXParserFileIngestModuleFactory." * - * @param canonicalName The canonical name. + * @param className The canonical class name. * - * @return The jython name or null if not in jython package. + * @return The Jython proxu class name or null if the extraction fails. */ - private static String getJythonName(String canonicalName) { - Matcher m = JYTHON_REGEX.matcher(canonicalName); + private static String getModuleNameFromJythonClassName(String className) { + Matcher m = JYTHON_MODULE_REGEX.matcher(className); if (m.find()) { - return String.format("%s.%s", m.group(1), m.group(2)); + return String.format("%s.%s", m.group(1), m.group(2)); //NON-NLS } else { return null; } } /** - * Adds a template to the appropriate map. If the class is a jython class, - * then it is added to the jython map. Otherwise, it is added to the - * mapping. + * Adds an ingest module template to one of two mappings of ingest module + * factory class names to module templates. One mapping is for ingest + * modules imnplemented using Java and the other is for ingest modules + * implemented using Jython. * - * @param mapping Mapping for non-jython objects. - * @param jythonMapping Mapping for jython objects. - * @param template The template to add. + * @param mapping Mapping for Java ingest module templates. + * @param jythonMapping Mapping for Jython ingest module templates. + * @param template The ingest module template. */ - private static void addModule(Map mapping, - Map jythonMapping, IngestModuleTemplate template) { - + private static void addModuleTemplateToSortingMap(Map mapping, Map jythonMapping, IngestModuleTemplate template) { String className = template.getModuleFactory().getClass().getCanonicalName(); - String jythonName = getJythonName(className); + String jythonName = getModuleNameFromJythonClassName(className); if (jythonName != null) { jythonMapping.put(jythonName, template); } else { @@ -307,432 +338,542 @@ final class IngestJobPipeline { } /** - * Creates the file and data source ingest pipelines. + * Creates the child ingest module pipelines for this ingest pipeline. + * + * @throws InterruptedException Exception thrown if the thread in which the + * pipeline is being created is interrupted. */ - private void createIngestPipelines() { - List ingestModuleTemplates = this.settings.getEnabledIngestModuleTemplates(); + private void createIngestModulePipelines() throws InterruptedException { + /* + * Get the enabled ingest module templates from the ingest job settings. + */ + List enabledTemplates = settings.getEnabledIngestModuleTemplates(); /** - * Make mappings of ingest module factory class names to templates. + * Sort the ingest module templates into buckets based on the module + * types the template can be used to create. A template may go into more + * than one bucket. Each bucket actually consists of two collections: + * one for Java modules and one for Jython modules. */ - Map dataSourceModuleTemplates = new LinkedHashMap<>(); - Map fileModuleTemplates = new LinkedHashMap<>(); - - // mappings for jython modules. These mappings are only used to determine modules in the pipelineconfig.xml. + Map javaDataSourceModuleTemplates = new LinkedHashMap<>(); Map jythonDataSourceModuleTemplates = new LinkedHashMap<>(); + Map javaFileModuleTemplates = new LinkedHashMap<>(); Map jythonFileModuleTemplates = new LinkedHashMap<>(); - - for (IngestModuleTemplate template : ingestModuleTemplates) { + Map javaArtifactModuleTemplates = new LinkedHashMap<>(); + Map jythonArtifactModuleTemplates = new LinkedHashMap<>(); + for (IngestModuleTemplate template : enabledTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - addModule(dataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); + addModuleTemplateToSortingMap(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); } if (template.isFileIngestModuleTemplate()) { - addModule(fileModuleTemplates, jythonFileModuleTemplates, template); + addModuleTemplateToSortingMap(javaFileModuleTemplates, jythonFileModuleTemplates, template); + } + if (template.isDataArtifactIngestModuleTemplate()) { + addModuleTemplateToSortingMap(javaArtifactModuleTemplates, jythonArtifactModuleTemplates, template); } } /** - * Use the mappings and the ingest pipelines configuration to create - * ordered lists of ingest module templates for each ingest pipeline. + * Take the module templates that have pipeline configuration entries + * out of the buckets and add them to ingest module pipeline templates + * in the order prescribed by the pipeline configuration. */ - IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates( - dataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - - List fileIngestModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates( - fileModuleTemplates, jythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - - List secondStageDataSourceModuleTemplates = IngestJobPipeline.getConfiguredIngestModuleTemplates( - dataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + IngestPipelinesConfiguration pipelineConfig = IngestPipelinesConfiguration.getInstance(); + List firstStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageOneDataSourceIngestPipelineConfig()); + List secondStageDataSourcePipelineTemplate = createIngestPipelineTemplate(javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfig.getStageTwoDataSourceIngestPipelineConfig()); + List filePipelineTemplate = createIngestPipelineTemplate(javaFileModuleTemplates, jythonFileModuleTemplates, pipelineConfig.getFileIngestPipelineConfig()); + List artifactPipelineTemplate = new ArrayList<>(); /** - * Add any module templates that were not specified in the pipelines - * configuration to an appropriate pipeline - either the first stage - * data source ingest pipeline or the file ingest pipeline. + * Add any ingest module templates remaining in the buckets to the + * appropriate ingest module pipeline templates. Data source level + * ingest modules templates that were not listed in the pipeline + * configuration are added to the first stage data source pipeline + * template, Java modules are added before Jython modules and Core + * Autopsy modules are added before third party modules. */ - addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates, jythonDataSourceModuleTemplates); - addOrdered(fileIngestModuleTemplates, fileModuleTemplates, jythonFileModuleTemplates); + addToIngestPipelineTemplate(firstStageDataSourcePipelineTemplate, javaDataSourceModuleTemplates, jythonDataSourceModuleTemplates); + addToIngestPipelineTemplate(filePipelineTemplate, javaFileModuleTemplates, jythonFileModuleTemplates); + addToIngestPipelineTemplate(artifactPipelineTemplate, javaArtifactModuleTemplates, jythonArtifactModuleTemplates); /** - * Construct the data source ingest pipelines. + * Construct the ingest module pipelines from the ingest module pipeline + * templates. */ - this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates); - this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates); - - /** - * Construct the file ingest pipelines, one per file ingest thread. - */ - try { - int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); - for (int i = 0; i < numberOfFileIngestThreads; ++i) { - FileIngestPipeline pipeline = new FileIngestPipeline(this, fileIngestModuleTemplates); - this.fileIngestPipelinesQueue.put(pipeline); - this.fileIngestPipelines.add(pipeline); - } - } catch (InterruptedException ex) { - /** - * The current thread was interrupted while blocked on a full queue. - * Blocking should actually never happen here, but reset the - * interrupted flag rather than just swallowing the exception. - */ - Thread.currentThread().interrupt(); - } - try { - SleuthkitCase skCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); - this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase); - this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); - } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest modules listing to case database", ex); - } - } - - private void addIngestModules(List templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException { - for (IngestModuleTemplate module : templates) { - ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber())); + firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourcePipelineTemplate); + secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourcePipelineTemplate); + int numberOfFileIngestThreads = IngestManager.getInstance().getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileIngestThreads; ++i) { + FileIngestPipeline pipeline = new FileIngestPipeline(this, filePipelineTemplate); + fileIngestPipelinesQueue.put(pipeline); + fileIngestPipelines.add(pipeline); } + artifactIngestPipeline = new DataArtifactIngestPipeline(this, artifactPipelineTemplate); } /** - * Uses an input collection of ingest module templates and a pipeline - * configuration, i.e., an ordered list of ingest module factory class - * names, to create an ordered output list of ingest module templates for an - * ingest pipeline. The ingest module templates are removed from the input - * collection as they are added to the output collection. + * Creates an ingest module pipeline template that can be used to construct + * an ingest module pipeline. * - * @param ingestModuleTemplates A mapping of ingest module factory - * class names to ingest module - * templates. - * @param jythonIngestModuleTemplates A mapping of jython processed class - * names to jython ingest module - * templates. - * @param pipelineConfig An ordered list of ingest module - * factory class names representing an - * ingest pipeline. + * @param javaIngestModuleTemplates Ingest module templates for ingest + * modules implemented using Java. + * @param jythonIngestModuleTemplates Ingest module templates for ingest + * modules implemented using Jython. + * @param pipelineConfig An ordered list of the ingest modules + * that belong in the ingest pipeline for + * which the template is being created. * - * @return An ordered list of ingest module templates, i.e., an - * uninstantiated pipeline. + * @return An ordered list of ingest module templates, i.e., a template for + * creating ingest module pipelines. */ - private static List getConfiguredIngestModuleTemplates( - Map ingestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { - List templates = new ArrayList<>(); + private static List createIngestPipelineTemplate(Map javaIngestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { + List pipelineTemplate = new ArrayList<>(); for (String moduleClassName : pipelineConfig) { - if (ingestModuleTemplates != null && ingestModuleTemplates.containsKey(moduleClassName)) { - templates.add(ingestModuleTemplates.remove(moduleClassName)); - } else if (jythonIngestModuleTemplates != null && jythonIngestModuleTemplates.containsKey(moduleClassName)) { - templates.add(jythonIngestModuleTemplates.remove(moduleClassName)); + if (javaIngestModuleTemplates.containsKey(moduleClassName)) { + pipelineTemplate.add(javaIngestModuleTemplates.remove(moduleClassName)); + } else if (jythonIngestModuleTemplates.containsKey(moduleClassName)) { + pipelineTemplate.add(jythonIngestModuleTemplates.remove(moduleClassName)); } } - return templates; + return pipelineTemplate; } /** - * Gets the identifier of this job. + * Gets the ID of this ingest pipeline. * - * @return The job identifier. + * @return The ID. */ long getId() { - return this.id; + return pipelineId; } /** - * Get the ingest execution context identifier. + * Gets the ingest execution context name. * - * @return The context string. + * @return The context name. */ String getExecutionContext() { - return this.settings.getExecutionContext(); + return settings.getExecutionContext(); } /** - * Gets the data source to be ingested by this job. + * Gets the data source to be analyzed by this ingest pipeline. * - * @return A Content object representing the data source. + * @return The data source. */ - Content getDataSource() { - return this.dataSource; + DataSource getDataSource() { + return dataSource; } /** - * Queries whether or not unallocated space should be processed as part of - * this job. + * Queries whether or not unallocated space should be processed by this + * ingest pipeline. * * @return True or false. */ boolean shouldProcessUnallocatedSpace() { - return this.settings.getProcessUnallocatedSpace(); + return settings.getProcessUnallocatedSpace(); } /** - * Gets the selected file ingest filter from settings. + * Gets the file ingest filter for this ingest pipeline. * - * @return True or false. + * @return The filter. */ FilesSet getFileIngestFilter() { - return this.settings.getFileFilter(); + return settings.getFileFilter(); } /** - * Checks to see if this job has at least one ingest pipeline. + * Checks to see if this ingest pipeline has at least one ingest module to + * run. * * @return True or false. */ - boolean hasIngestPipeline() { - return this.hasFirstStageDataSourceIngestPipeline() - || this.hasFileIngestPipeline() - || this.hasSecondStageDataSourceIngestPipeline(); + boolean hasIngestModules() { + return hasFileIngestModules() + || hasFirstStageDataSourceIngestModules() + || hasSecondStageDataSourceIngestModules() + || hasDataArtifactIngestModules(); } /** - * Checks to see if this job has a first stage data source level ingest - * pipeline. + * Checks to see if this ingest pipeline has at least one ingest module to + * run. * * @return True or false. */ - private boolean hasFirstStageDataSourceIngestPipeline() { - return (this.firstStageDataSourceIngestPipeline.isEmpty() == false); + boolean hasDataSourceIngestModules() { + if (stage == Stages.SECOND_STAGE) { + return hasSecondStageDataSourceIngestModules(); + } else { + return hasFirstStageDataSourceIngestModules(); + } } /** - * Checks to see if this job has a second stage data source level ingest - * pipeline. + * Checks to see if this ingest pipeline has at least one first stage data + * source level ingest module to run. * * @return True or false. */ - private boolean hasSecondStageDataSourceIngestPipeline() { - return (this.secondStageDataSourceIngestPipeline.isEmpty() == false); + private boolean hasFirstStageDataSourceIngestModules() { + return (firstStageDataSourceIngestPipeline.isEmpty() == false); } /** - * Checks to see if this job has a file level ingest pipeline. + * Checks to see if this ingest pipeline has at least one second stage data + * source level ingest module to run. * * @return True or false. */ - private boolean hasFileIngestPipeline() { - if (!this.fileIngestPipelines.isEmpty()) { - return !this.fileIngestPipelines.get(0).isEmpty(); + private boolean hasSecondStageDataSourceIngestModules() { + return (secondStageDataSourceIngestPipeline.isEmpty() == false); + } + + /** + * Checks to see if this ingest pipeline has at least one file ingest module + * to run. + * + * @return True or false. + */ + boolean hasFileIngestModules() { + if (!fileIngestPipelines.isEmpty()) { + return !fileIngestPipelines.get(0).isEmpty(); } return false; } /** - * Starts up the ingest pipelines for this job. + * Checks to see if this ingest pipeline has at least one data artifact + * ingest module to run. + * + * @return True or false. + */ + boolean hasDataArtifactIngestModules() { + return (artifactIngestPipeline.isEmpty() == false); + } + + /** + * Starts up this ingest pipeline. * * @return A collection of ingest module startup errors, empty on success. */ - List start() { - if (dataSource == null) { - // TODO - Remove once data source is always present during initialization - throw new IllegalStateException("Ingest started before setting data source"); - } - List errors = startUpIngestPipelines(); + List startUp() { + List errors = startUpIngestModulePipelines(); if (errors.isEmpty()) { - try { - this.ingestJob = Case.getCurrentCaseThrows().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); - } catch (TskCoreException | NoCurrentCaseException ex) { - logErrorMessage(Level.WARNING, "Failed to add ingest job info to case database", ex); //NON-NLS - } - - if (this.hasFirstStageDataSourceIngestPipeline() || this.hasFileIngestPipeline()) { - if (ingestMode == IngestJob.Mode.BATCH) { - logInfoMessage("Starting first stage analysis"); //NON-NLS - this.startFirstStage(); + recordIngestJobStartUpInfo(); + if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + if (parentJob.getIngestMode() == IngestJob.Mode.STREAMING) { + startFirstStageInStreamingMode(); } else { - logInfoMessage("Preparing for first stage analysis"); //NON-NLS - this.startFileIngestStreaming(); + startFirstStageInBatchMode(); } - } else if (this.hasSecondStageDataSourceIngestPipeline()) { - logInfoMessage("Starting second stage analysis"); //NON-NLS - this.startSecondStage(); + } else if (hasSecondStageDataSourceIngestModules()) { + startSecondStage(); } } return errors; } /** - * Starts up each of the ingest pipelines for this job to collect any file - * and data source level ingest modules errors that might occur. + * Writes start up data about the ingest job into the case database. The + * case database returns an object that is retained to allow the additon of + * a completion time when the ingest job is finished. + */ + void recordIngestJobStartUpInfo() { + try { + SleuthkitCase caseDb = Case.getCurrentCase().getSleuthkitCase(); + List ingestModuleInfoList = new ArrayList<>(); + for (IngestModuleTemplate module : settings.getEnabledIngestModuleTemplates()) { + IngestModuleType moduleType = getIngestModuleTemplateType(module); + IngestModuleInfo moduleInfo = caseDb.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), moduleType, module.getModuleFactory().getModuleVersionNumber()); + ingestModuleInfoList.add(moduleInfo); + } + ingestJobInfo = caseDb.addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModuleInfoList, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); + } catch (TskCoreException ex) { + logErrorMessage(Level.SEVERE, "Failed to add ingest job info to case database", ex); //NON-NLS + } + } + + /** + * Determines the type of ingest modules a given ingest module template + * supports. * - * @return A collection of ingest module startup errors, empty on success. + * @param moduleTemplate The ingest module template. + * + * @return The ingest module type, may be IngestModuleType.MULTIPLE. */ - private List startUpIngestPipelines() { - List errors = new ArrayList<>(); - - /* - * Start the data-source-level ingest module pipelines. - */ - errors.addAll(this.firstStageDataSourceIngestPipeline.startUp()); - errors.addAll(this.secondStageDataSourceIngestPipeline.startUp()); - - /* - * If the data-source-level ingest pipelines were successfully started, - * start the file-level ingest pipelines (one per pipeline file ingest - * thread). - */ - if (errors.isEmpty()) { - for (FileIngestPipeline pipeline : this.fileIngestPipelinesQueue) { - errors.addAll(pipeline.startUp()); - if (!errors.isEmpty()) { - /* - * If there are start up errors, the ingest job will not - * proceed, so shut down any file ingest pipelines that did - * start up. - */ - while (!this.fileIngestPipelinesQueue.isEmpty()) { - FileIngestPipeline startedPipeline = this.fileIngestPipelinesQueue.poll(); - if (startedPipeline.isRunning()) { - List shutDownErrors = startedPipeline.shutDown(); - if (!shutDownErrors.isEmpty()) { - /* - * The start up errors will ultimately be - * reported to the user for possible remedy, but - * the shut down errors are logged here. - */ - logIngestModuleErrors(shutDownErrors); - } - } - } - break; - } + private IngestModuleType getIngestModuleTemplateType(IngestModuleTemplate moduleTemplate) { + IngestModuleType type = null; + if (moduleTemplate.isDataSourceIngestModuleTemplate()) { + type = IngestModuleType.DATA_SOURCE_LEVEL; + } + if (moduleTemplate.isFileIngestModuleTemplate()) { + if (type == null) { + type = IngestModuleType.FILE_LEVEL; + } else { + type = IngestModuleType.MULTIPLE; } - } + } + if (moduleTemplate.isDataArtifactIngestModuleTemplate()) { + if (type == null) { + type = IngestModuleType.DATA_ARTIFACT; + } else { + type = IngestModuleType.MULTIPLE; + } + } + return type; + } + + /** + * Starts up each of the child ingest module pipelines in this ingest + * pipeline. + * + * Note that all of the child pipelines are started so that any and all + * start up errors can be returned to the caller. It is important to capture + * all of the errors, because the ingest job will be automatically cancelled + * and the errors will be reported to the user so either the issues can be + * addressed or the modules that can't start up can be disabled before the + * ingest job is attempted again. + * + * @return A list of ingest module startup errors, empty on success. + */ + private List startUpIngestModulePipelines() { + List errors = new ArrayList<>(); + errors.addAll(startUpIngestModulePipeline(firstStageDataSourceIngestPipeline)); + errors.addAll(startUpIngestModulePipeline(secondStageDataSourceIngestPipeline)); + for (FileIngestPipeline pipeline : fileIngestPipelines) { + List filePipelineErrors = startUpIngestModulePipeline(pipeline); + if (!filePipelineErrors.isEmpty()) { + /* + * If one file pipeline copy can't start up, assume that none of + * them will be able to start up for the same reason. + */ + errors.addAll(filePipelineErrors); + break; + } + } + errors.addAll(startUpIngestModulePipeline(artifactIngestPipeline)); return errors; } /** - * Starts the first stage of this job. + * Starts up an ingest module pipeline. If there are any start up errors, + * the pipeline is immediately shut down. + * + * @param pipeline The ingest task pipeline to start up. + * + * @return A list of ingest module startup errors, empty on success. */ - private void startFirstStage() { - this.stage = IngestJobPipeline.Stages.FIRST_STAGE_FILES_AND_DATASOURCE; - - if (this.hasFileIngestPipeline()) { - synchronized (this.fileIngestProgressLock) { - this.estimatedFilesToProcess = this.dataSource.accept(new GetFilesCountVisitor()); + private List startUpIngestModulePipeline(IngestTaskPipeline pipeline) { + List startUpErrors = pipeline.startUp(); + if (!startUpErrors.isEmpty()) { + List shutDownErrors = pipeline.shutDown(); + if (!shutDownErrors.isEmpty()) { + logIngestModuleErrors(shutDownErrors); } } + return startUpErrors; + } - if (this.doUI) { - /** - * Start one or both of the first stage ingest progress bars. + /** + * Starts the first stage of this pipeline in batch mode. In batch mode, all + * of the files in the data source (excepting carved and derived files) have + * already been added to the case database by the data source processor. + */ + private void startFirstStageInBatchMode() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in batch mode"); //NON-NLS + stage = Stages.FIRST_STAGE; + + /* + * Do a count of the files the data source processor has added to + * the case database. This estimate will be used for ingest progress + * snapshots and for the file ingest progress bar if running with a + * GUI. */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); + if (hasFileIngestModules()) { + long filesToProcess; + if (files.isEmpty()) { + filesToProcess = dataSource.accept(new GetFilesCountVisitor()); + } else { + filesToProcess = files.size(); + } + synchronized (fileIngestProgressLock) { + estimatedFilesToProcess = filesToProcess; + } } - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); - } - } - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - /** - * Schedule the first stage tasks. - */ - if (this.hasFirstStageDataSourceIngestPipeline() && this.hasFileIngestPipeline()) { - logInfoMessage("Scheduling first stage data source and file level analysis tasks"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleIngestTasks(this); - } else if (this.hasFirstStageDataSourceIngestPipeline()) { - logInfoMessage("Scheduling first stage data source level analysis tasks"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); - } else { - logInfoMessage("Scheduling file level analysis tasks, no first stage data source level analysis configured"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleFileIngestTasks(this, this.files); - - /** - * No data source ingest task has been scheduled for this stage, and - * it is possible, if unlikely, that no file ingest tasks were - * actually scheduled since there are files that get filtered out by - * the tasks scheduler. In this special case, an ingest thread will - * never get to check for completion of this stage of the job, so do - * it now. + /* + * If running with a GUI, start ingest progress bars in the lower + * right hand corner of the main application window. */ - this.checkForStageCompleted(); + if (doUI) { + if (hasFileIngestModules()) { + startFileIngestProgressBar(); + } + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } + } + + /* + * Make the first stage data source level ingest pipeline the + * current data source level pipeline. + */ + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + + /* + * Schedule the first stage ingest tasks and then immediately check + * for stage completion. This is necessary because it is possible + * that zero tasks will actually make it to task execution due to + * the file filter or other ingest job settings. In that case, there + * will never be a stage completion check in an ingest thread + * executing an ingest task, so such a job would run forever without + * a check here. + */ + if (!files.isEmpty() && hasFileIngestModules()) { + taskScheduler.scheduleFileIngestTasks(this, files); + } else if (hasFirstStageDataSourceIngestModules() || hasFileIngestModules() || hasDataArtifactIngestModules()) { + taskScheduler.scheduleIngestTasks(this); + } + checkForStageCompleted(); } } /** - * Prepares for file ingest. Used for streaming ingest. Does not schedule - * any file tasks - those will come from calls to addStreamingIngestFiles(). + * Starts the first stage of this pipeline in streaming mode. In streaming + * mode, the data source processor streams files into the pipeline as it + * adds them to the case database and file level analysis can begin before + * data source level analysis. */ - private void startFileIngestStreaming() { - synchronized (this.stageCompletionCheckLock) { - this.stage = IngestJobPipeline.Stages.FIRST_STAGE_FILES_ONLY; - } + private void startFirstStageInStreamingMode() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting first stage analysis in streaming mode"); //NON-NLS + stage = Stages.FIRST_STAGE_STREAMING; - if (this.hasFileIngestPipeline()) { - synchronized (this.fileIngestProgressLock) { - this.estimatedFilesToProcess = 0; // Set to indeterminate until the data source is complete + if (doUI) { + /* + * If running with a GUI, start ingest progress bars in the + * lower right hand corner of the main application window. + */ + if (hasFileIngestModules()) { + /* + * Note that because estimated files remaining to process + * still has its initial value of zero, the progress bar + * will start in the "indeterminate" state. An estimate of + * the files to process can be computed in + */ + startFileIngestProgressBar(); + } + if (hasDataArtifactIngestModules()) { + startArtifactIngestProgressBar(); + } } - } - if (this.doUI) { - if (this.hasFileIngestPipeline()) { - this.startFileIngestProgressBar(); + if (hasDataArtifactIngestModules()) { + /* + * Schedule artifact ingest tasks for any artifacts currently in + * the case database. This needs to be done before any files or + * the data source are streamed in to avoid analyzing data + * artifacts added to the case database by the data source level + * or file level ingest tasks. + */ + taskScheduler.scheduleDataArtifactIngestTasks(this); } } - - logInfoMessage("Waiting for streaming files"); //NON-NLS - } - - /** - * Start data source ingest. Used for streaming ingest when the data source - * is not ready when ingest starts. - */ - private void startDataSourceIngestStreaming() { - - // Now that the data source is complete, we can get the estimated number of - // files and switch to a determinate progress bar. - synchronized (fileIngestProgressLock) { - if (null != this.fileIngestProgress) { - estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); - fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess); - } - } - - if (this.doUI) { - /** - * Start the first stage data source ingest progress bar. - */ - if (this.hasFirstStageDataSourceIngestPipeline()) { - this.startDataSourceIngestProgressBar(); - } - } - - /** - * Make the first stage data source level ingest pipeline the current - * data source level pipeline. - */ - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.firstStageDataSourceIngestPipeline; - } - - logInfoMessage("Scheduling first stage data source level analysis tasks"); //NON-NLS - synchronized (this.stageCompletionCheckLock) { - this.stage = IngestJobPipeline.Stages.FIRST_STAGE_FILES_AND_DATASOURCE; - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); - } } /** - * Starts the second stage of this ingest job. + * Notifies the ingest pipeline running in streaming mode that the data + * source is now ready for analysis. + */ + void addStreamedDataSource() { + synchronized (stageTransitionLock) { + logInfoMessage("Starting full first stage analysis in streaming mode"); //NON-NLS + stage = IngestJobPipeline.Stages.FIRST_STAGE; + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + + if (hasFileIngestModules()) { + /* + * Do a count of the files the data source processor has added + * to the case database. This estimate will be used for ingest + * progress snapshots and for the file ingest progress bar if + * running with a GUI. + */ + long filesToProcess = dataSource.accept(new GetFilesCountVisitor()); + synchronized (fileIngestProgressLock) { + estimatedFilesToProcess = filesToProcess; + if (doUI && fileIngestProgressBar != null) { + fileIngestProgressBar.switchToDeterminate((int) estimatedFilesToProcess); + } + } + } + + if (doUI) { + if (hasFirstStageDataSourceIngestModules()) { + startDataSourceIngestProgressBar(); + } + } + + currentDataSourceIngestPipeline = firstStageDataSourceIngestPipeline; + if (hasFirstStageDataSourceIngestModules()) { + IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); + } else { + /* + * If no data source level ingest task is scheduled at this time + * and all of the file level and artifact ingest tasks scheduled + * during the initial file streaming stage have already + * executed, there will never be a stage completion check in an + * ingest thread executing an ingest task, so such a job would + * run forever without a check here. + */ + checkForStageCompleted(); + } + } + } + + /** + * Starts the second stage ingest task pipelines. */ private void startSecondStage() { - logInfoMessage("Starting second stage analysis"); //NON-NLS - this.stage = IngestJobPipeline.Stages.SECOND_STAGE; - if (this.doUI) { - this.startDataSourceIngestProgressBar(); + synchronized (stageTransitionLock) { + if (hasSecondStageDataSourceIngestModules()) { + logInfoMessage(String.format("Starting second stage ingest task pipelines for %s (objID=%d, jobID=%d)", dataSource.getName(), dataSource.getId(), parentJob.getId())); //NON-NLS + stage = IngestJobPipeline.Stages.SECOND_STAGE; + + if (doUI) { + startDataSourceIngestProgressBar(); + } + + currentDataSourceIngestPipeline = secondStageDataSourceIngestPipeline; + taskScheduler.scheduleDataSourceIngestTask(this); + } } - synchronized (this.dataSourceIngestPipelineLock) { - this.currentDataSourceIngestPipeline = this.secondStageDataSourceIngestPipeline; + } + + /** + * Starts a progress bar for the results ingest tasks for the ingest job. + */ + private void startArtifactIngestProgressBar() { + if (doUI) { + synchronized (artifactIngestProgressLock) { + String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataArtifactIngest.displayName", this.dataSource.getName()); + artifactIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + IngestJobPipeline.this.cancel(IngestJob.CancellationReason.USER_CANCELLED); + return true; + } + }); + artifactIngestProgressBar.start(); + artifactIngestProgressBar.switchToIndeterminate(); + } } - logInfoMessage("Scheduling second stage data source level analysis tasks"); //NON-NLS - IngestJobPipeline.taskScheduler.scheduleDataSourceIngestTask(this); } /** @@ -744,7 +885,7 @@ final class IngestJobPipeline { String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", this.dataSource.getName()); - this.dataSourceIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { + this.dataSourceIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { // If this method is called, the user has already pressed @@ -764,8 +905,8 @@ final class IngestJobPipeline { return true; } }); - this.dataSourceIngestProgress.start(); - this.dataSourceIngestProgress.switchToIndeterminate(); + this.dataSourceIngestProgressBar.start(); + this.dataSourceIngestProgressBar.switchToIndeterminate(); } } } @@ -779,7 +920,7 @@ final class IngestJobPipeline { String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName()); - this.fileIngestProgress = ProgressHandle.createHandle(displayName, new Cancellable() { + this.fileIngestProgressBar = ProgressHandle.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { // If this method is called, the user has already pressed @@ -790,8 +931,8 @@ final class IngestJobPipeline { return true; } }); - this.fileIngestProgress.start(); - this.fileIngestProgress.switchToDeterminate((int) this.estimatedFilesToProcess); + this.fileIngestProgressBar.start(); + this.fileIngestProgressBar.switchToDeterminate((int) this.estimatedFilesToProcess); } } } @@ -801,49 +942,17 @@ final class IngestJobPipeline { * completed and does a stage transition if they are. */ private void checkForStageCompleted() { - if (ingestMode == IngestJob.Mode.BATCH) { - checkForStageCompletedBatch(); - } else { - checkForStageCompletedStreaming(); - } - } - - /** - * Checks to see if the ingest tasks for the current stage of this job are - * completed and does a stage transition if they are. - */ - private void checkForStageCompletedBatch() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJobPipeline.taskScheduler.currentTasksAreCompleted(this)) { - switch (this.stage) { - case FIRST_STAGE_FILES_AND_DATASOURCE: - this.finishFirstStage(); - break; - case SECOND_STAGE: - this.finish(); - break; - } + synchronized (stageTransitionLock) { + if (stage == Stages.FIRST_STAGE_STREAMING) { + return; } - } - } - - /** - * Checks to see if the ingest tasks for the current stage of this job are - * completed and does a stage transition if they are. - */ - private void checkForStageCompletedStreaming() { - synchronized (this.stageCompletionCheckLock) { - if (IngestJobPipeline.taskScheduler.currentTasksAreCompleted(this)) { - switch (this.stage) { - case FIRST_STAGE_FILES_ONLY: - // Nothing to do here - need to wait for the data source - break; - case FIRST_STAGE_FILES_AND_DATASOURCE: - // Finish file and data source ingest, start second stage (if applicable) - this.finishFirstStage(); + if (taskScheduler.currentTasksAreCompleted(this)) { + switch (stage) { + case FIRST_STAGE: + finishFirstStage(); break; case SECOND_STAGE: - this.finish(); + shutDown(); break; } } @@ -855,256 +964,295 @@ final class IngestJobPipeline { * job and starts the second stage, if appropriate. */ private void finishFirstStage() { - logInfoMessage("Finished first stage analysis"); //NON-NLS + synchronized (stageTransitionLock) { + logInfoMessage("Finished first stage analysis"); //NON-NLS - // Shut down the file ingest pipelines. Note that no shut down is - // required for the data source ingest pipeline because data source - // ingest modules do not have a shutdown() method. - List errors = new ArrayList<>(); - while (!this.fileIngestPipelinesQueue.isEmpty()) { - FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); - if (pipeline.isRunning()) { - errors.addAll(pipeline.shutDown()); + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); + while (!fileIngestPipelinesQueue.isEmpty()) { + FileIngestPipeline pipeline = fileIngestPipelinesQueue.poll(); + shutDownIngestModulePipeline(pipeline); } - } - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } - if (this.doUI) { - // Finish the first stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + } + + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } } } - // Finish the file ingest progress bar, if it hasn't already - // been finished. - synchronized (this.fileIngestProgressLock) { - if (this.fileIngestProgress != null) { - this.fileIngestProgress.finish(); - this.fileIngestProgress = null; - } - } - } - - /** - * Start the second stage, if appropriate. - */ - if (!this.cancelled && this.hasSecondStageDataSourceIngestPipeline()) { - this.startSecondStage(); - } else { - this.finish(); - } - } - - /** - * Shuts down the ingest pipelines and progress bars for this job. - */ - private void finish() { - logInfoMessage("Finished analysis"); //NON-NLS - this.stage = IngestJobPipeline.Stages.FINALIZATION; - - if (this.doUI) { - // Finish the second stage data source ingest progress bar, if it hasn't - // already been finished. - synchronized (this.dataSourceIngestProgressLock) { - if (this.dataSourceIngestProgress != null) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } - if (ingestJob != null) { - if (this.cancelled) { - try { - ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } + if (!cancelled && hasSecondStageDataSourceIngestModules()) { + startSecondStage(); } else { - try { - ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); - } - } - try { - this.ingestJob.setEndDateTime(new Date()); - } catch (TskCoreException ex) { - logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); + shutDown(); } } - this.parentJob.ingestJobPipelineFinished(this); } /** - * Passes the data source for this job through the currently active data - * source level ingest pipeline. + * Shuts down the ingest module pipelines and progress bars for this job. + */ + private void shutDown() { + synchronized (stageTransitionLock) { + logInfoMessage("Finished all tasks"); //NON-NLS + stage = IngestJobPipeline.Stages.FINALIZATION; + + shutDownIngestModulePipeline(currentDataSourceIngestPipeline); + shutDownIngestModulePipeline(artifactIngestPipeline); + + if (doUI) { + synchronized (dataSourceIngestProgressLock) { + if (dataSourceIngestProgressBar != null) { + dataSourceIngestProgressBar.finish(); + dataSourceIngestProgressBar = null; + } + } + + synchronized (fileIngestProgressLock) { + if (fileIngestProgressBar != null) { + fileIngestProgressBar.finish(); + fileIngestProgressBar = null; + } + } + + synchronized (artifactIngestProgressLock) { + if (artifactIngestProgressBar != null) { + artifactIngestProgressBar.finish(); + artifactIngestProgressBar = null; + } + } + } + + if (ingestJobInfo != null) { + if (cancelled) { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.CANCELLED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } + } else { + try { + ingestJobInfo.setIngestJobStatus(IngestJobStatusType.COMPLETED); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to update ingest job status in case database", ex); + } + } + try { + ingestJobInfo.setEndDateTime(new Date()); + } catch (TskCoreException ex) { + logErrorMessage(Level.WARNING, "Failed to set job end date in case database", ex); + } + } + } + + parentJob.notifyIngestPipelineShutDown(this); + } + + /** + * Shuts down an ingest task pipeline. + * + * @param pipeline The pipeline. + */ + private void shutDownIngestModulePipeline(IngestTaskPipeline pipeline) { + if (pipeline.isRunning()) { + List errors = new ArrayList<>(); + errors.addAll(pipeline.shutDown()); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + } + } + + /** + * Passes the data source for the ingest job through the currently active + * data source level ingest task pipeline (first stage or second stage data + * source ingest modules). * * @param task A data source ingest task wrapping the data source. */ - void process(DataSourceIngestTask task) { + void execute(DataSourceIngestTask task) { try { - synchronized (this.dataSourceIngestPipelineLock) { - if (!this.isCancelled() && !this.currentDataSourceIngestPipeline.isEmpty()) { - List errors = new ArrayList<>(); - errors.addAll(this.currentDataSourceIngestPipeline.performTask(task)); - if (!errors.isEmpty()) { - logIngestModuleErrors(errors); - } + if (!isCancelled()) { + List errors = new ArrayList<>(); + errors.addAll(currentDataSourceIngestPipeline.executeTask(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); } } - - if (this.doUI) { - /** - * Shut down the data source ingest progress bar right away. - * Data source-level processing is finished for this stage. - */ - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; - } - } - } - } finally { - IngestJobPipeline.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); + taskScheduler.notifyTaskCompleted(task); + checkForStageCompleted(); } } /** - * Passes a file from the data source for this job through the file level - * ingest pipeline. + * Passes a file from the data source for the ingest job through the file + * ingest task pipeline (file ingest modules). * - * @param task A file ingest task. - * - * @throws InterruptedException if the thread executing this code is - * interrupted while blocked on taking from or - * putting to the file ingest pipelines - * collection. + * @param task A file ingest task wrapping the file. */ - void process(FileIngestTask task) throws InterruptedException { + void execute(FileIngestTask task) { try { - if (!this.isCancelled()) { - FileIngestPipeline pipeline = this.fileIngestPipelinesQueue.take(); + if (!isCancelled()) { + FileIngestPipeline pipeline = fileIngestPipelinesQueue.take(); if (!pipeline.isEmpty()) { + /* + * Get the file from the task. If the file was "streamed," + * the task may only have the file object ID and a trip to + * the case database will be required. + */ AbstractFile file; try { file = task.getFile(); } catch (TskCoreException ex) { - // In practice, this task would never have been enqueued since the file - // lookup would have failed there. List errors = new ArrayList<>(); - errors.add(new IngestModuleError("Ingest Job Pipeline", ex)); + errors.add(new IngestModuleError("Ingest Pipeline", ex)); logIngestModuleErrors(errors); - this.fileIngestPipelinesQueue.put(pipeline); + fileIngestPipelinesQueue.put(pipeline); return; } - synchronized (this.fileIngestProgressLock) { - ++this.processedFiles; - if (this.doUI) { - /** - * Update the file ingest progress bar. - */ - if (this.processedFiles <= this.estimatedFilesToProcess) { - this.fileIngestProgress.progress(file.getName(), (int) this.processedFiles); + synchronized (fileIngestProgressLock) { + ++processedFiles; + if (doUI) { + if (processedFiles <= estimatedFilesToProcess) { + fileIngestProgressBar.progress(file.getName(), (int) processedFiles); } else { - this.fileIngestProgress.progress(file.getName(), (int) this.estimatedFilesToProcess); + fileIngestProgressBar.progress(file.getName(), (int) estimatedFilesToProcess); } - this.filesInProgress.add(file.getName()); + filesInProgress.add(file.getName()); } } /** - * Run the file through the pipeline. + * Run the file through the modules in the pipeline. */ List errors = new ArrayList<>(); - errors.addAll(pipeline.performTask(task)); + errors.addAll(pipeline.executeTask(task)); if (!errors.isEmpty()) { logIngestModuleErrors(errors, file); } - if (this.doUI && !this.cancelled) { - synchronized (this.fileIngestProgressLock) { + if (doUI && !cancelled) { + synchronized (fileIngestProgressLock) { /** * Update the file ingest progress bar again, in * case the file was being displayed. */ - this.filesInProgress.remove(file.getName()); - if (this.filesInProgress.size() > 0) { - this.fileIngestProgress.progress(this.filesInProgress.get(0)); + filesInProgress.remove(file.getName()); + if (filesInProgress.size() > 0) { + fileIngestProgressBar.progress(filesInProgress.get(0)); } else { - this.fileIngestProgress.progress(""); + fileIngestProgressBar.progress(""); } } } } - this.fileIngestPipelinesQueue.put(pipeline); + fileIngestPipelinesQueue.put(pipeline); + } + } catch (InterruptedException ex) { + logger.log(Level.SEVERE, String.format("Unexpected interrupt of file ingest thread during execution of file ingest job (file obj ID = %d)", task.getFileId()), ex); + Thread.currentThread().interrupt(); // Reset thread interrupted flag + } finally { + taskScheduler.notifyTaskCompleted(task); + checkForStageCompleted(); + } + } + + /** + * Passes a data artifact from the data source for the ingest job through + * the data artifact ingest task pipeline (data artifact ingest modules). + * + * @param task A data artifact ingest task wrapping the file. + */ + void execute(DataArtifactIngestTask task) { + try { + if (!isCancelled() && !artifactIngestPipeline.isEmpty()) { + List errors = new ArrayList<>(); + errors.addAll(artifactIngestPipeline.executeTask(task)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } } } finally { - IngestJobPipeline.taskScheduler.notifyTaskCompleted(task); - this.checkForStageCompleted(); + taskScheduler.notifyTaskCompleted(task); + checkForStageCompleted(); } } /** - * Add a list of files (by object ID) to the ingest queue. Must call start() - * prior to adding files. + * Adds some subset of the streamed files for a streaming mode ingest job to + * this pipeline. * - * @param fileObjIds List of newly added file IDs. + * @param fileObjIds The object IDs of the files. */ - void addStreamingIngestFiles(List fileObjIds) { - - // Return if there are no file ingest modules enabled. - if (!hasFileIngestPipeline()) { - return; - } - - if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY)) { - IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); - } else { - logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + void addStreamedFiles(List fileObjIds) { + if (hasFileIngestModules()) { + if (stage.equals(Stages.FIRST_STAGE_STREAMING)) { + IngestJobPipeline.taskScheduler.scheduleStreamedFileIngestTasks(this, fileObjIds); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } } } /** - * Starts data source ingest. Should be called after the data source - * processor has finished (i.e., all files are in the database) - */ - void processStreamingIngestDataSource() { - startDataSourceIngestStreaming(); - checkForStageCompleted(); - } - - /** - * Adds more files from the data source for this job to the job, e.g., adds - * extracted or carved files. Not currently supported for the second stage - * of the job. + * Adds additional files (e.g., extracted or carved files) for any type of + * ingest job to this pipeline after startUp() has been called. Not + * currently supported for second stage of the job. * * @param files A list of the files to add. */ void addFiles(List files) { - if (stage.equals(Stages.FIRST_STAGE_FILES_ONLY) - || stage.equals(Stages.FIRST_STAGE_FILES_AND_DATASOURCE)) { - IngestJobPipeline.taskScheduler.fastTrackFileIngestTasks(this, files); + if (stage.equals(Stages.FIRST_STAGE_STREAMING) + || stage.equals(Stages.FIRST_STAGE)) { + taskScheduler.fastTrackFileIngestTasks(this, files); } else { - logErrorMessage(Level.SEVERE, "Adding files to job during second stage analysis not supported"); + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); } /** * The intended clients of this method are ingest modules running code - * on an ingest thread that is holding a reference to an ingest task, in - * which case a completion check would not be necessary, so this is a - * bit of defensive programming. + * in an ingest thread that is holding a reference to a "primary" ingest + * task that was the source of the files, in which case a completion + * check would not be necessary, so this is a bit of defensive + * programming. */ - this.checkForStageCompleted(); + checkForStageCompleted(); + } + + /** + * Adds data artifacts for any type of ingest job to this pipeline after + * startUp() has been called. + * + * @param artifacts + */ + void addDataArtifacts(List artifacts) { + List artifactsToAnalyze = new ArrayList<>(artifacts); + if (stage.equals(Stages.FIRST_STAGE_STREAMING) + || stage.equals(Stages.FIRST_STAGE) + || stage.equals(Stages.SECOND_STAGE)) { + taskScheduler.scheduleDataArtifactIngestTasks(this, artifactsToAnalyze); + } else { + logErrorMessage(Level.SEVERE, "Adding streaming files to job during stage " + stage.toString() + " not supported"); + } + + /** + * The intended clients of this method are ingest modules running code + * in an ingest thread that is holding a reference to a "primary" ingest + * task that was the source of the files, in which case a completion + * check would not be necessary, so this is a bit of defensive + * programming. + */ + checkForStageCompleted(); } /** @@ -1116,7 +1264,7 @@ final class IngestJobPipeline { void updateDataSourceIngestProgressBarDisplayName(String displayName) { if (this.doUI && !this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.setDisplayName(displayName); + this.dataSourceIngestProgressBar.setDisplayName(displayName); } } } @@ -1132,8 +1280,8 @@ final class IngestJobPipeline { void switchDataSourceIngestProgressBarToDeterminate(int workUnits) { if (this.doUI && !this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToDeterminate(workUnits); + if (null != this.dataSourceIngestProgressBar) { + this.dataSourceIngestProgressBar.switchToDeterminate(workUnits); } } } @@ -1147,8 +1295,8 @@ final class IngestJobPipeline { void switchDataSourceIngestProgressBarToIndeterminate() { if (this.doUI && !this.cancelled) { synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.switchToIndeterminate(); + if (null != this.dataSourceIngestProgressBar) { + this.dataSourceIngestProgressBar.switchToIndeterminate(); } } } @@ -1161,10 +1309,10 @@ final class IngestJobPipeline { * @param workUnits Number of work units performed. */ void advanceDataSourceIngestProgressBar(int workUnits) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress("", workUnits); + if (doUI && !cancelled) { + synchronized (dataSourceIngestProgressLock) { + if (null != dataSourceIngestProgressBar) { + dataSourceIngestProgressBar.progress("", workUnits); } } } @@ -1177,10 +1325,10 @@ final class IngestJobPipeline { * @param currentTask The task name. */ void advanceDataSourceIngestProgressBar(String currentTask) { - if (this.doUI && !this.cancelled) { - synchronized (this.dataSourceIngestProgressLock) { - if (null != this.dataSourceIngestProgress) { - this.dataSourceIngestProgress.progress(currentTask); + if (doUI && !cancelled) { + synchronized (dataSourceIngestProgressLock) { + if (null != dataSourceIngestProgressBar) { + dataSourceIngestProgressBar.progress(currentTask); } } } @@ -1197,7 +1345,7 @@ final class IngestJobPipeline { void advanceDataSourceIngestProgressBar(String currentTask, int workUnits) { if (this.doUI && !this.cancelled) { synchronized (this.fileIngestProgressLock) { - this.dataSourceIngestProgress.progress(currentTask, workUnits); + this.dataSourceIngestProgressBar.progress(currentTask, workUnits); } } } @@ -1232,8 +1380,8 @@ final class IngestJobPipeline { * is pressed. */ synchronized (this.dataSourceIngestProgressLock) { - this.dataSourceIngestProgress.finish(); - this.dataSourceIngestProgress = null; + this.dataSourceIngestProgressBar.finish(); + this.dataSourceIngestProgressBar = null; this.startDataSourceIngestProgressBar(); } } @@ -1245,7 +1393,7 @@ final class IngestJobPipeline { * @return The currently running module, may be null. */ DataSourceIngestPipeline.DataSourcePipelineModule getCurrentDataSourceIngestModule() { - if (null != currentDataSourceIngestPipeline) { + if (currentDataSourceIngestPipeline != null) { return (DataSourceIngestPipeline.DataSourcePipelineModule) currentDataSourceIngestPipeline.getCurrentlyRunningModule(); } else { return null; @@ -1269,20 +1417,20 @@ final class IngestJobPipeline { void cancel(IngestJob.CancellationReason reason) { this.cancelled = true; this.cancellationReason = reason; - IngestJobPipeline.taskScheduler.cancelPendingTasksForIngestJob(this); + IngestJobPipeline.taskScheduler.cancelPendingFileTasksForIngestJob(this); if (this.doUI) { synchronized (this.dataSourceIngestProgressLock) { - if (null != dataSourceIngestProgress) { - dataSourceIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", this.dataSource.getName())); - dataSourceIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); + if (null != dataSourceIngestProgressBar) { + dataSourceIngestProgressBar.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.initialDisplayName", this.dataSource.getName())); + dataSourceIngestProgressBar.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); } } synchronized (this.fileIngestProgressLock) { - if (null != this.fileIngestProgress) { - this.fileIngestProgress.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName())); - this.fileIngestProgress.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); + if (null != this.fileIngestProgressBar) { + this.fileIngestProgressBar.setDisplayName(NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", this.dataSource.getName())); + this.fileIngestProgressBar.progress(NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling")); } } } @@ -1293,23 +1441,11 @@ final class IngestJobPipeline { } pausedIngestThreads.clear(); } - + // If a data source had no tasks in progress it may now be complete. checkForStageCompleted(); } - /** - * Set the current module name being run and the file name it is running on. - * To be used for more detailed cancelling. - * - * @param moduleName Name of module currently running. - * @param taskName Name of file the module is running on. - */ - void setCurrentFileIngestModule(String moduleName, String taskName) { - this.currentFileIngestModule = moduleName; - this.currentFileIngestTask = taskName; - } - /** * Queries whether or not cancellation, i.e., a shutdown of the data source * level and file level ingest pipelines for this job, has been requested. @@ -1336,7 +1472,7 @@ final class IngestJobPipeline { * @param message The message. */ private void logInfoMessage(String message) { - logger.log(Level.INFO, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId())); //NON-NLS + logger.log(Level.INFO, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJobInfo.getIngestJobId())); //NON-NLS } /** @@ -1348,7 +1484,7 @@ final class IngestJobPipeline { * @param throwable The throwable associated with the error. */ private void logErrorMessage(Level level, String message, Throwable throwable) { - logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId()), throwable); //NON-NLS + logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id = %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJobInfo.getIngestJobId()), throwable); //NON-NLS } /** @@ -1359,7 +1495,7 @@ final class IngestJobPipeline { * @param message The message. */ private void logErrorMessage(Level level, String message) { - logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id %d)", message, this.dataSource.getName(), this.dataSource.getId(), id, ingestJob.getIngestJobId())); //NON-NLS + logger.log(level, String.format("%s (data source = %s, objId = %d, pipeline id = %d, ingest job id %d)", message, this.dataSource.getName(), this.dataSource.getId(), pipelineId, ingestJobInfo.getIngestJobId())); //NON-NLS } /** @@ -1386,7 +1522,7 @@ final class IngestJobPipeline { } /** - * Gets a snapshot of this jobs state and performance. + * Gets a snapshot of this ingest pipelines current state. * * @return An ingest job statistics object. */ @@ -1398,7 +1534,6 @@ final class IngestJobPipeline { */ boolean fileIngestRunning = false; Date fileIngestStartTime = null; - for (FileIngestPipeline pipeline : this.fileIngestPipelines) { if (pipeline.isRunning()) { fileIngestRunning = true; @@ -1413,19 +1548,19 @@ final class IngestJobPipeline { long estimatedFilesToProcessCount = 0; long snapShotTime = new Date().getTime(); IngestJobTasksSnapshot tasksSnapshot = null; - if (getIngestTasksSnapshot) { synchronized (fileIngestProgressLock) { processedFilesCount = this.processedFiles; estimatedFilesToProcessCount = this.estimatedFilesToProcess; snapShotTime = new Date().getTime(); } - tasksSnapshot = IngestJobPipeline.taskScheduler.getTasksSnapshotForJob(id); - + tasksSnapshot = taskScheduler.getTasksSnapshotForJob(pipelineId); } - return new Snapshot(this.dataSource.getName(), id, createTime, - getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime, + return new Snapshot(dataSource.getName(), + pipelineId, createTime, + getCurrentDataSourceIngestModule(), + fileIngestRunning, fileIngestStartTime, cancelled, cancellationReason, cancelledDataSourceIngestModules, processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java index 918c30760f..7de83ded4c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2018 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -318,7 +318,7 @@ public final class IngestJobSettings { // Add modules that are going to be used for this ingest depending on type. for (IngestModuleFactory moduleFactory : allModuleFactories) { - if (this.ingestType.equals(IngestType.ALL_MODULES)) { + if (moduleFactory.isDataArtifactIngestModuleFactory() || ingestType.equals(IngestType.ALL_MODULES)) { moduleFactories.add(moduleFactory); } else if (this.ingestType.equals(IngestType.DATA_SOURCE_ONLY) && moduleFactory.isDataSourceIngestModuleFactory()) { moduleFactories.add(moduleFactory); @@ -330,7 +330,7 @@ public final class IngestJobSettings { for (IngestModuleFactory moduleFactory : moduleFactories) { loadedModuleNames.add(moduleFactory.getModuleDisplayName()); } - + /** * Hard coding Plaso to be disabled by default. loadedModuleNames is * passed below as the default list of enabled modules so briefly remove @@ -361,7 +361,7 @@ public final class IngestJobSettings { if (plasoLoaded) { loadedModuleNames.add(plasoModuleName); } - + /** * Check for missing modules and create warnings if any are found. */ @@ -549,7 +549,7 @@ public final class IngestJobSettings { * @return The file path. */ private String getModuleSettingsFilePath(IngestModuleFactory factory) { - String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT; + String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT; Path path = Paths.get(this.moduleSettingsFolderPath, fileName); return path.toAbsolutePath().toString(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index bb6a8991e0..f6dea4c510 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.ingest; +import com.google.common.eventbus.Subscribe; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.awt.EventQueue; import java.beans.PropertyChangeEvent; @@ -68,8 +69,11 @@ import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisStartedEvent; import org.sleuthkit.autopsy.ingest.events.FileAnalyzedEvent; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard; +import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** @@ -129,8 +133,9 @@ public class IngestManager implements IngestProgressSnapshotProvider { private final Map> startIngestJobFutures = new ConcurrentHashMap<>(); @GuardedBy("ingestJobsById") private final Map ingestJobsById = new HashMap<>(); - private final ExecutorService dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS; + private final ExecutorService dataSourceLevelIngestJobTasksExecutor; private final ExecutorService fileLevelIngestJobTasksExecutor; + private final ExecutorService resultIngestTasksExecutor; private final ExecutorService eventPublishingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-ingest-events-%d").build()); //NON-NLS; private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ServicesMonitor servicesMonitor = ServicesMonitor.getInstance(); @@ -168,6 +173,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { * source level ingest job tasks to the data source level ingest job * tasks executor. */ + dataSourceLevelIngestJobTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-data-source-ingest-%d").build()); //NON-NLS; long threadId = nextIngestManagerTaskId.incrementAndGet(); dataSourceLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); @@ -184,6 +190,13 @@ public class IngestManager implements IngestProgressSnapshotProvider { fileLevelIngestJobTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue())); ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); } + + resultIngestTasksExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("IM-results-ingest-%d").build()); //NON-NLS; + threadId = nextIngestManagerTaskId.incrementAndGet(); + resultIngestTasksExecutor.submit(new ExecuteIngestJobTasksTask(threadId, IngestTasksScheduler.getInstance().getResultIngestTaskQueue())); + // RJCTODO + // ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId)); + // RJCTODO: Where is the shut down code? } /** @@ -248,9 +261,10 @@ public class IngestManager implements IngestProgressSnapshotProvider { }); } - /* - * Handles a current case opened event by clearing the ingest messages inbox - * and opening a remote event channel for the current case. + /** + * Handles a current case opened event by clearing the ingest messages + * inbox, opening a remote event channel for the current case, and + * registering to receive events from the event bus for the case database. * * Note that current case change events are published in a strictly * serialized manner, i.e., one event at a time, synchronously. @@ -265,6 +279,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { jobEventPublisher.openRemoteEventChannel(String.format(INGEST_JOB_EVENT_CHANNEL_NAME, channelPrefix)); moduleEventPublisher.openRemoteEventChannel(String.format(INGEST_MODULE_EVENT_CHANNEL_NAME, channelPrefix)); } + openedCase.getSleuthkitCase().registerForEvents(this); } catch (NoCurrentCaseException | AutopsyEventException ex) { logger.log(Level.SEVERE, "Failed to open remote events channel", ex); //NON-NLS MessageNotifyUtil.Notify.error(NbBundle.getMessage(IngestManager.class, "IngestManager.OpenEventChannel.Fail.Title"), @@ -272,10 +287,27 @@ public class IngestManager implements IngestProgressSnapshotProvider { } } - /* + /** + * Handles artifacts posted events published by the Sleuth Kit layer + * blackboard via the event bus for the case database. + * + * @param tskEvent A Sleuth Kit data model ArtifactsPostedEvent from the + * case database event bus. + */ + @Subscribe + void handleArtifactsPosted(Blackboard.ArtifactsPostedEvent tskEvent) { + for (BlackboardArtifact.Type artifactType : tskEvent.getArtifactTypes()) { + ModuleDataEvent legacyEvent = new ModuleDataEvent(tskEvent.getModuleName(), artifactType, tskEvent.getArtifacts(artifactType)); + AutopsyEvent autopsyEvent = new BlackboardPostEvent(legacyEvent); + eventPublishingExecutor.submit(new PublishEventTask(autopsyEvent, moduleEventPublisher)); + } + } + + /** * Handles a current case closed event by cancelling all ingest jobs for the - * case, closing the remote event channel for the case, and clearing the - * ingest messages inbox. + * case, unregistering from receiving events from the case database, closing + * the remote event channel for the case, and clearing the ingest messages + * inbox. * * Note that current case change events are published in a strictly * serialized manner, i.e., one event at a time, synchronously. @@ -285,7 +317,8 @@ public class IngestManager implements IngestProgressSnapshotProvider { * TODO (JIRA-2227): IngestManager should wait for cancelled ingest jobs * to complete when a case is closed. */ - this.cancelAllIngestJobs(IngestJob.CancellationReason.CASE_CLOSED); + cancelAllIngestJobs(IngestJob.CancellationReason.CASE_CLOSED); + Case.getCurrentCase().getSleuthkitCase().unregisterForEvents(this); jobEventPublisher.closeRemoteEventChannel(); moduleEventPublisher.closeRemoteEventChannel(); caseIsOpen = false; @@ -455,7 +488,11 @@ public class IngestManager implements IngestProgressSnapshotProvider { ingestJobsById.put(job.getId(), job); } IngestManager.logger.log(Level.INFO, "Starting ingest job {0}", job.getId()); //NON-NLS - errors = job.start(); + try { + errors = job.start(); + } catch (InterruptedException ex) { + return new IngestJobStartResult(null, new IngestManagerException("Interrupted while starting ingest", ex), errors); //NON-NLS + } if (errors.isEmpty()) { this.fireIngestJobStarted(job.getId()); } else { @@ -492,7 +529,8 @@ public class IngestManager implements IngestProgressSnapshotProvider { * * @param job The completed job. */ - void finishIngestJob(IngestJob job) { + void finishIngestJob(IngestJob job + ) { long jobId = job.getId(); synchronized (ingestJobsById) { ingestJobsById.remove(jobId); @@ -700,18 +738,6 @@ public class IngestManager implements IngestProgressSnapshotProvider { eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher)); } - /** - * Publishes an ingest module event signifying a blackboard post by an - * ingest module. - * - * @param moduleDataEvent A ModuleDataEvent with the details of the - * blackboard post. - */ - void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { - AutopsyEvent event = new BlackboardPostEvent(moduleDataEvent); - eventPublishingExecutor.submit(new PublishEventTask(event, moduleEventPublisher)); - } - /** * Publishes an ingest module event signifying discovery of additional * content by an ingest module. diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java index 8bfeef57fb..ec83a129c1 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModule.java @@ -74,7 +74,7 @@ public interface IngestModule { * ingest module instance is discarded. The module should respond by doing * things like releasing private resources, submitting final results, and * posting a final ingest message. - * + * * IMPORTANT: If the module instances must share resources, the modules are * responsible for synchronizing access to the shared resources and doing * reference counting as required to release those resources correctly. @@ -87,7 +87,8 @@ public interface IngestModule { } /** - * An exception for the use of ingest modules. + * An exception for ingest modules to throw if they experience a start up + * error. */ public class IngestModuleException extends Exception { @@ -108,7 +109,7 @@ public interface IngestModule { } /** - * A return code for subclass process() methods. + * A return code for process() method implementations. */ public enum ProcessResult { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java index 1ce3770360..84a5cb6bfd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleError.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java index 5b522d552a..e473086d18 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactory.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -91,14 +91,13 @@ public interface IngestModuleFactory { * family of ingest modules the factory creates. For example, the Autopsy * core hash lookup ingest module factory provides a global settings panel * to import and create hash databases. The hash databases are then enabled - * or disabled per ingest job using an ingest job settings panel. If the - * module family does not have global settings, the factory may extend - * IngestModuleFactoryAdapter to get an implementation of this method that - * returns false. + * or disabled per ingest job using an ingest job settings panel. * * @return True if the factory provides a global settings panel. */ - boolean hasGlobalSettingsPanel(); + default boolean hasGlobalSettingsPanel() { + return false; + } /** * Gets a user interface panel that allows a user to change settings that @@ -106,68 +105,64 @@ public interface IngestModuleFactory { * creates. For example, the Autopsy core hash lookup ingest module factory * provides a global settings panel to import and create hash databases. The * imported hash databases are then enabled or disabled per ingest job using - * ingest an ingest job settings panel. If the module family does not have a - * global settings, the factory may extend IngestModuleFactoryAdapter to get - * an implementation of this method that throws an - * UnsupportedOperationException. + * ingest an ingest job settings panel. * * @return A global settings panel. */ - IngestModuleGlobalSettingsPanel getGlobalSettingsPanel(); + default IngestModuleGlobalSettingsPanel getGlobalSettingsPanel() { + throw new UnsupportedOperationException(); + } /** * Gets the default per ingest job settings for instances of the family of * ingest modules the factory creates. For example, the Autopsy core hash * lookup ingest modules family uses hash databases imported or created * using its global settings panel. All of the hash databases are enabled by - * default for an ingest job. If the module family does not have per ingest - * job settings, the factory may extend IngestModuleFactoryAdapter to get an - * implementation of this method that returns an instance of the - * NoIngestModuleJobSettings class. + * default for an ingest job. * * @return The default ingest job settings. */ - IngestModuleIngestJobSettings getDefaultIngestJobSettings(); + default IngestModuleIngestJobSettings getDefaultIngestJobSettings() { + return new NoIngestModuleIngestJobSettings(); + } /** * Queries the factory to determine if it provides user a interface panel to * allow a user to make per ingest job settings for instances of the family * of ingest modules the factory creates. For example, the Autopsy core hash * lookup ingest module factory provides an ingest job settings panels to - * enable or disable hash databases per ingest job. If the module family - * does not have per ingest job settings, the factory may extend - * IngestModuleFactoryAdapter to get an implementation of this method that - * returns false. + * enable or disable hash databases per ingest job. * * @return True if the factory provides ingest job settings panels. */ - boolean hasIngestJobSettingsPanel(); + default boolean hasIngestJobSettingsPanel() { + return false; + } /** * Gets a user interface panel that can be used to set per ingest job * settings for instances of the family of ingest modules the factory * creates. For example, the core hash lookup ingest module factory provides * an ingest job settings panel to enable or disable hash databases per - * ingest job. If the module family does not have per ingest job settings, - * the factory may extend IngestModuleFactoryAdapter to get an - * implementation of this method that throws an - * UnsupportedOperationException. + * ingest job. * * @param settings Per ingest job settings to initialize the panel. * * @return An ingest job settings panel. */ - IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings); + default IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { + throw new UnsupportedOperationException(); + } /** * Queries the factory to determine if it is capable of creating data source - * ingest modules. If the module family does not include data source ingest - * modules, the factory may extend IngestModuleFactoryAdapter to get an - * implementation of this method that returns false. + * ingest modules. * * @return True if the factory can create data source ingest modules. */ - boolean isDataSourceIngestModuleFactory(); + default boolean isDataSourceIngestModuleFactory() { + return false; + } /** * Creates a data source ingest module instance. @@ -189,26 +184,24 @@ public interface IngestModuleFactory { * correctly. Also, more than one ingest job may be in progress at any given * time. This must also be taken into consideration when sharing resources * between module instances. modules. - *

- * If the module family does not include data source ingest modules, the - * factory may extend IngestModuleFactoryAdapter to get an implementation of - * this method that throws an UnsupportedOperationException. * - * @param settings The settings for the ingest job. + * @param ingestOptions The settings for the ingest job. * * @return A data source ingest module instance. */ - DataSourceIngestModule createDataSourceIngestModule(IngestModuleIngestJobSettings settings); + default DataSourceIngestModule createDataSourceIngestModule(IngestModuleIngestJobSettings ingestOptions) { + throw new UnsupportedOperationException(); + } /** * Queries the factory to determine if it is capable of creating file ingest - * modules. If the module family does not include file ingest modules, the - * factory may extend IngestModuleFactoryAdapter to get an implementation of - * this method that returns false. + * modules. * * @return True if the factory can create file ingest modules. */ - boolean isFileIngestModuleFactory(); + default boolean isFileIngestModuleFactory() { + return false; + } /** * Creates a file ingest module instance. @@ -230,14 +223,52 @@ public interface IngestModuleFactory { * correctly. Also, more than one ingest job may be in progress at any given * time. This must also be taken into consideration when sharing resources * between module instances. modules. - *

- * If the module family does not include file ingest modules, the factory - * may extend IngestModuleFactoryAdapter to get an implementation of this - * method that throws an UnsupportedOperationException. * * @param settings The settings for the ingest job. * * @return A file ingest module instance. */ - FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings); + default FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { + throw new UnsupportedOperationException(); + } + + /** + * Queries the factory to determine if it is capable of creating data + * artifact ingest modules. + * + * @return True or false. + */ + default boolean isDataArtifactIngestModuleFactory() { + return false; + } + + /** + * Creates a data artifact ingest module instance. + *

+ * Autopsy will generally use the factory to several instances of each type + * of module for each ingest job it performs. Completing an ingest job + * entails processing a single data source (e.g., a disk image) and all of + * the files from the data source, including files extracted from archives + * and any unallocated space (made to look like a series of files). The data + * source is passed through one or more pipelines of data source ingest + * modules. The files are passed through one or more pipelines of file + * ingest modules. + *

+ * The ingest framework may use multiple threads to complete an ingest job, + * but it is guaranteed that there will be no more than one module instance + * per thread. However, if the module instances must share resources, the + * modules are responsible for synchronizing access to the shared resources + * and doing reference counting as required to release those resources + * correctly. Also, more than one ingest job may be in progress at any given + * time. This must also be taken into consideration when sharing resources + * between module instances. modules. + * + * @param settings The settings for the ingest job. + * + * @return A file ingest module instance. + */ + default DataArtifactIngestModule createDataArtifactIngestModule(IngestModuleIngestJobSettings settings) { + throw new UnsupportedOperationException(); + } + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java index 46a7427f02..c4d01e8bce 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryAdapter.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,10 @@ package org.sleuthkit.autopsy.ingest; /** * An adapter that provides no-op implementations of various IngestModuleFactory * methods. + * + * NOTE: As of Java 8, interfaces can have default methods. IngestModuleFactory + * now provides default no-op versions of all of its optional methods. This + * class is no longer needed and can be DEPRECATED when convenient. */ public abstract class IngestModuleFactoryAdapter implements IngestModuleFactory { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java index 83a290c78a..26285f6439 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestModuleTemplate.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2018 Basis Technology Corp. + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -85,6 +85,14 @@ public final class IngestModuleTemplate { return moduleFactory.createFileIngestModule(settings); } + public boolean isDataArtifactIngestModuleTemplate() { + return moduleFactory.isDataArtifactIngestModuleFactory(); + } + + public DataArtifactIngestModule createDataArtifactIngestModule() { + return moduleFactory.createDataArtifactIngestModule(settings); + } + public void setEnabled(boolean enabled) { this.enabled = enabled; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java index 5568b0f3e4..b15f5723b5 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2014-2018 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -165,26 +165,22 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { private class IngestJobTableModel extends AbstractTableModel { - private final String[] columnNames = {NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.dataSource"), + private static final long serialVersionUID = 1L; + + private final String[] columnNames = { + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.jobID"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dataSource"), NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.start"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.numProcessed"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.filesPerSec"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.inProgress"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.filesQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.dirQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.rootQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.streamingQueued"), - NbBundle.getMessage(this.getClass(), - "IngestJobTableModel.colName.dsQueued")}; + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.numProcessed"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesPerSec"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.inProgress"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.filesQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dirQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.rootQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.streamingQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.dsQueued"), + NbBundle.getMessage(this.getClass(), "IngestJobTableModel.colName.artifactsQueued")}; + private List jobSnapshots; private IngestJobTableModel() { @@ -250,6 +246,9 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { case 10: cellValue = snapShot.getDsQueueSize(); break; + case 11: + cellValue = snapShot.getArtifactTasksQueueSize(); + break; default: cellValue = null; break; @@ -260,6 +259,8 @@ class IngestProgressSnapshotPanel extends javax.swing.JPanel { private class ModuleTableModel extends AbstractTableModel { + private static final long serialVersionUID = 1L; + private class ModuleStats implements Comparable { private final String name; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java index 7bad9ebf98..e2dd585582 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2018 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,10 +19,12 @@ package org.sleuthkit.autopsy.ingest; import java.util.Map; +import java.util.logging.Level; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ModuleSettings; +import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.SleuthkitCase; /** @@ -31,6 +33,7 @@ import org.sleuthkit.datamodel.SleuthkitCase; */ public final class IngestServices { + private static Logger logger = Logger.getLogger(IngestServices.class.getName()); private static IngestServices instance = null; /** @@ -105,11 +108,17 @@ public final class IngestServices { * @param moduleDataEvent A module data event, i.e., an event that * encapsulates artifact data. * - * @deprecated use org.sleuthkit.datamodel.Blackboard.postArtifact instead. + * @deprecated Use org.sleuthkit.datamodel.Blackboard.postArtifact or + * org.sleuthkit.datamodel.Blackboard.postArtifacts instead. */ @Deprecated public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { - IngestManager.getInstance().fireIngestModuleDataEvent(moduleDataEvent); + try { + Blackboard blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard(); + blackboard.postArtifacts(moduleDataEvent.getArtifacts(), moduleDataEvent.getModuleName()); + } catch (NoCurrentCaseException | Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, "Failed to post artifacts", ex); + } } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java index ebfabf2ab2..41c3736986 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTask.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * - * Copyright 2014 Basis Technology Corp. + * + * Copyright 2014-2021 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,32 +20,76 @@ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.datamodel.Content; +/** + * An ingest task that will be executed by an ingest thread using a given ingest + * job pipeline. Three examples of concrete types of ingest tasks are tasks to + * analyze a data source, tasks to analyze the files in a data source, and tasks + * that analyze data artifacts. + */ abstract class IngestTask { private final static long NOT_SET = Long.MIN_VALUE; private final IngestJobPipeline ingestJobPipeline; private long threadId; + /** + * Constructs an ingest task that will be executed by an ingest thread using + * a given ingest job pipeline. Three examples of concrete types of ingest + * tasks are tasks to analyze a data source, tasks to analyze the files in a + * data source, and tasks that analyze data artifacts. + * + * @param ingestJobPipeline The ingest job pipeline to use to execute the + * task. + */ IngestTask(IngestJobPipeline ingestJobPipeline) { this.ingestJobPipeline = ingestJobPipeline; threadId = NOT_SET; } + /** + * Gets the ingest job pipeline used to complete this task. + * + * @return The ingest job pipeline. + */ IngestJobPipeline getIngestJobPipeline() { return ingestJobPipeline; } + /** + * Gets the data source for the ingest job of which this task is a part. + * + * @return The data source. + */ Content getDataSource() { return getIngestJobPipeline().getDataSource(); } + /** + * Gets the thread ID of the ingest thread executing this task. + * + * @return The thread ID. + */ long getThreadId() { return threadId; } + /** + * Sets the thread ID of the ingest thread executing this task. + * + * @param threadId The thread ID. + */ void setThreadId(long threadId) { this.threadId = threadId; } - abstract void execute(long threadId) throws InterruptedException; + /** + * Records the ingest thread ID of the calling thread and executes this task + * using the ingest job pipeline specified when the task was created. The + * implementation of the method should simple call + * super.setThreadId(threadId) and getIngestJobPipeline().process(this). + * + * @param threadId The numeric ID of the ingest thread executing this task. + */ + abstract void execute(long threadId); + } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java index aafbdb14fa..026063462c 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTaskPipeline.java @@ -26,6 +26,8 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Optional; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import org.openide.util.NbBundle; @@ -33,54 +35,65 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; /** - * An abstract superclass for pipelines of ingest modules for a given ingest - * task type. Some examples of ingest task types: data source level ingest - * tasks, file ingest tasks, data artifact ingest tasks, etc. Subclasses need to - * implement a specialization of the inner PipelineModule abstract superclass - * for the type of ingest modules that make up the pipeline. + * An abstract superclass for pipelines of ingest modules that execute ingest + * tasks for an ingest job. + * + * Conceptually, an ingest job pipeline is divided into one or more "sub + * pipelines" that are actually ingest task pipelines of varying types. Thus, + * the type parameter of this generic is an ingest task type. + * + * IMPORTANT: Subclasses need to both extend this class, and to implement a + * specialization of the inner PipelineModule abstract superclass. * * @param The ingest task type. */ +@ThreadSafe abstract class IngestTaskPipeline { private static final Logger logger = Logger.getLogger(IngestTaskPipeline.class.getName()); private final IngestJobPipeline ingestJobPipeline; + @GuardedBy("this") private final List moduleTemplates; + @GuardedBy("this") private final List> modules; private volatile Date startTime; private volatile boolean running; private volatile PipelineModule currentModule; /** - * Constructs an instance of an abstract superclass for pipelines of ingest - * modules for a given ingest task type. Some examples of ingest task types: - * data source level ingest tasks, file ingest tasks, data artifact ingest - * tasks, etc. Subclasses need to implement a specialization of the inner - * PipelineModule abstract superclass for the type of ingest modules that - * make up the pipeline. + * Constructs the superclass part of a pipeline of ingest modules that + * executes ingest tasks for an ingest job. * - * @param ingestJobPipeline The ingest job pipeline that owns this pipeline. - * @param moduleTemplates The ingest module templates that define this - * pipeline. + * @param ingestPipeline The parent ingest job pipeline for this ingest + * task pipeline. + * @param moduleTemplates The ingest module templates that define this + * ingest task pipeline. May be an empty list. */ - IngestTaskPipeline(IngestJobPipeline ingestJobPipeline, List moduleTemplates) { - this.ingestJobPipeline = ingestJobPipeline; + IngestTaskPipeline(IngestJobPipeline ingestPipeline, List moduleTemplates) { + this.ingestJobPipeline = ingestPipeline; + /* + * The creation of ingest modules from the ingest module templates has + * been deliberately deferred to the startUp() method so that any and + * all errors in module construction or start up can be reported to the + * client code. + */ this.moduleTemplates = moduleTemplates; modules = new ArrayList<>(); } /** - * Indicates whether or not there are any ingest modules in this pipeline. + * Indicates whether or not there are any ingest modules in this ingest task + * pipeline. * * @return True or false. */ - boolean isEmpty() { + synchronized boolean isEmpty() { return modules.isEmpty(); } /** - * Queries whether or not this pipeline is running, i.e., started and not - * shut down. + * Queries whether or not this ingest task pipeline is running, i.e., the + * startUp() method has been called and the shutDown() has not been called. * * @return True or false. */ @@ -89,56 +102,77 @@ abstract class IngestTaskPipeline { } /** - * Starts up the ingest modules in this pipeline. + * Starts up this ingest task pipeline by calling the startUp() methods of + * the ingest modules in the pipeline. * - * @return A list of ingest module startup errors, possibly empty. + * @return A list of ingest module start up errors, possibly empty. */ - List startUp() { - createIngestModules(moduleTemplates); - return startUpIngestModules(); + synchronized List startUp() { + List errors = new ArrayList<>(); + if (!running) { + /* + * The creation of ingest modules from the ingest module templates + * has been deliberately deferred to the startUp() method so that + * any and all errors in module construction or start up can be + * reported to the client code. + */ + createIngestModules(moduleTemplates); + errors.addAll(startUpIngestModules()); + } else { + errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestTaskPipelineException("Pipeline already started"))); //NON-NLS + } + return errors; } /** - * Creates the ingest modules for this pipeline. + * Creates the ingest modules for this ingest task pipeline from the given + * ingest module templates. * - * @param moduleTemplates The ingest module templates avaialble to this - * pipeline. + * @param moduleTemplates The ingest module templates. */ private void createIngestModules(List moduleTemplates) { - for (IngestModuleTemplate template : moduleTemplates) { - Optional> module = acceptModuleTemplate(template); - if (module.isPresent()) { - modules.add(module.get()); + if (modules.isEmpty()) { + for (IngestModuleTemplate template : moduleTemplates) { + Optional> module = acceptModuleTemplate(template); + if (module.isPresent()) { + modules.add(module.get()); + } } } } /** - * Determines if the type of ingest module that can be created from a given - * ingest module template should be added to this pipeline. If so, the - * ingest module is created and returned. + * Determines if one of the types of ingest modules that can be created from + * a given ingest module template should be added to this ingest task + * pipeline. If so, the ingest module is created and returned. * - * @param ingestModuleTemplate The ingest module template to be used or - * ignored, as appropriate to the pipeline type. + * @param template The ingest module template to be used or ignored, as + * appropriate to the pipeline type. * - * @return An Optional that is either empty or contains a newly created and - * wrapped ingest module. + * @return An Optional that is either empty or contains a newly created + * ingest module of type T, wrapped in a PipelineModule decorator. */ - abstract Optional> acceptModuleTemplate(IngestModuleTemplate ingestModuleTemplate); + abstract Optional> acceptModuleTemplate(IngestModuleTemplate template); /** - * Starts up the ingest modules in the pipeline. + * Starts up the ingest modules in this ingest task pipeline. * - * @return A list of ingest module startup errors, possibly empty. + * @return A list of ingest module start up errors, possibly empty. */ private List startUpIngestModules() { + List errors = new ArrayList<>(); startTime = new Date(); running = true; - List errors = new ArrayList<>(); for (PipelineModule module : modules) { try { module.startUp(new IngestJobContext(ingestJobPipeline)); - } catch (Throwable ex) { // Catch-all exception firewall + } catch (Throwable ex) { + /* + * A catch-all exception firewall. Start up errors for all of + * the ingest modules, whether checked exceptions or runtime + * exceptions, are reported to allow correction of all of the + * error conditions in one go. + */ errors.add(new IngestModuleError(module.getDisplayName(), ex)); } } @@ -146,7 +180,7 @@ abstract class IngestTaskPipeline { } /** - * Returns the start up time of this pipeline. + * Returns the start up time of this ingest task pipeline. * * @return The file processing start time, may be null if this pipeline has * not been started yet. @@ -160,56 +194,57 @@ abstract class IngestTaskPipeline { } /** - * Does any preparation required before performing a task. - * - * @param task The task. - * - * @throws IngestTaskPipelineException Thrown if there is an error preparing - * to perform the task. - */ - abstract void prepareTask(T task) throws IngestTaskPipelineException; - - /** - * Performs an ingest task using the ingest modules in this pipeline. + * Executes an ingest task by calling the process() methods of the ingest + * modules in this ingest task pipeline. * * @param task The task. * * @return A list of ingest module task processing errors, possibly empty. */ - List performTask(T task) { + synchronized List executeTask(T task) { List errors = new ArrayList<>(); - if (!this.ingestJobPipeline.isCancelled()) { - pauseIfScheduled(); - if (ingestJobPipeline.isCancelled()) { - return errors; - } - try { - prepareTask(task); - } catch (IngestTaskPipelineException ex) { - errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS - return errors; - } - for (PipelineModule module : modules) { + if (running) { + if (!ingestJobPipeline.isCancelled()) { pauseIfScheduled(); if (ingestJobPipeline.isCancelled()) { - break; + return errors; } try { - currentModule = module; - currentModule.setProcessingStartTime(); - module.performTask(ingestJobPipeline, task); - } catch (Throwable ex) { // Catch-all exception firewall - errors.add(new IngestModuleError(module.getDisplayName(), ex)); + prepareForTask(task); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + return errors; } - if (ingestJobPipeline.isCancelled()) { - break; + for (PipelineModule module : modules) { + pauseIfScheduled(); + if (ingestJobPipeline.isCancelled()) { + break; + } + try { + currentModule = module; + currentModule.setProcessingStartTime(); + module.executeTask(ingestJobPipeline, task); + } catch (Throwable ex) { + /* + * A catch-all exception firewall. Note that a runtime + * exception from a single module does not stop + * processing of the task by the other modules in the + * pipeline. + */ + errors.add(new IngestModuleError(module.getDisplayName(), ex)); + } + if (ingestJobPipeline.isCancelled()) { + break; + } } } - } - try { - completeTask(task); - } catch (IngestTaskPipelineException ex) { - errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + try { + cleanUpAfterTask(task); + } catch (IngestTaskPipelineException ex) { + errors.add(new IngestModuleError("Ingest Task Pipeline", ex)); //NON-NLS + } + } else { + errors.add(new IngestModuleError("Ingest Task Pipeline", new IngestTaskPipelineException("Pipeline not started or shut down"))); //NON-NLS } currentModule = null; return errors; @@ -265,7 +300,18 @@ abstract class IngestTaskPipeline { } /** - * Gets the currently running module. + * Does any task type specific preparation required before executing an + * ingest task. + * + * @param task The task. + * + * @throws IngestTaskPipelineException Thrown if there is an error preparing + * to execute the task. + */ + abstract void prepareForTask(T task) throws IngestTaskPipelineException; + + /** + * Gets the currently running ingest module. * * @return The module, possibly null if no module is currently running. */ @@ -274,21 +320,11 @@ abstract class IngestTaskPipeline { } /** - * Does any clean up required after performing a task. - * - * @param task The task. - * - * @throws IngestTaskPipelineException Thrown if there is an error cleaning - * up after performing the task. - */ - abstract void completeTask(T task) throws IngestTaskPipelineException; - - /** - * Shuts down all of the modules in the pipeline. + * Shuts down all of the ingest modules in this pipeline. * * @return A list of shut down errors, possibly empty. */ - List shutDown() { + synchronized List shutDown() { List errors = new ArrayList<>(); if (running == true) { for (PipelineModule module : modules) { @@ -315,8 +351,22 @@ abstract class IngestTaskPipeline { } /** - * An abstract superclass for a wrapper that adds ingest infrastructure + * Does any task type specific clean up required after executing an ingest + * task. + * + * @param task The task. + * + * @throws IngestTaskPipelineException Thrown if there is an error cleaning + * up after performing the task. + */ + abstract void cleanUpAfterTask(T task) throws IngestTaskPipelineException; + + /** + * An abstract superclass for a decorator that adds ingest infrastructure * operations to an ingest module. + * + * IMPORTANT: Subclasses of IngestTaskPipeline need to implement a + * specialization this class */ static abstract class PipelineModule implements IngestModule { @@ -325,7 +375,7 @@ abstract class IngestTaskPipeline { private volatile Date processingStartTime; /** - * Constructs an instance of an abstract superclass for a wrapper that + * Constructs an instance of an abstract superclass for a decorator that * adds ingest infrastructure operations to an ingest module. * * @param module The ingest module to be wrapped. @@ -338,7 +388,7 @@ abstract class IngestTaskPipeline { } /** - * Gets the class name of the wrapped ingest module. + * Gets the class name of the decorated ingest module. * * @return The class name. */ @@ -347,7 +397,7 @@ abstract class IngestTaskPipeline { } /** - * Gets the display name of the wrapped ingest module. + * Gets the display name of the decorated ingest module. * * @return The display name. */ @@ -356,7 +406,7 @@ abstract class IngestTaskPipeline { } /** - * Sets the processing start time for the wrapped module to the system + * Sets the processing start time for the decorated module to the system * time when this method is called. */ void setProcessingStartTime() { @@ -364,7 +414,7 @@ abstract class IngestTaskPipeline { } /** - * Gets the the processing start time for the wrapped module. + * Gets the the processing start time for the decorated module. * * @return The start time, will be null if the module has not started * processing the data source yet. @@ -379,16 +429,17 @@ abstract class IngestTaskPipeline { } /** - * Performs an ingest task. + * Executes an ingest task using the process() method of the decorated + * module. * * @param ingestJobPipeline The ingest job pipeline that owns the ingest - * module pipeline this module belongs to. - * @param task The task to process. + * task pipeline this module belongs to. + * @param task The task to execute. * - * @throws IngestModuleException Excepton thrown if there is an error + * @throws IngestModuleException Exception thrown if there is an error * performing the task. */ - abstract void performTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException; + abstract void executeTask(IngestJobPipeline ingestJobPipeline, T task) throws IngestModuleException; @Override public void shutDown() { @@ -398,19 +449,31 @@ abstract class IngestTaskPipeline { } /** - * An exception for the use of ingest task pipelines. + * An exception thrown by an ingest task pipeline. */ public static class IngestTaskPipelineException extends Exception { private static final long serialVersionUID = 1L; + /** + * Constructs an exception to be thrown by an ingest task pipeline. + * + * @param message The exception message. + */ public IngestTaskPipelineException(String message) { super(message); } + /** + * Constructs an exception to be thrown by an ingest task pipeline. + * + * @param message The exception message. + * @param cause The exception cause. + */ public IngestTaskPipelineException(String message, Throwable cause) { super(message, cause); } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java index 2ec96915cc..1492d6d824 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2018 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,16 +36,20 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataArtifact; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.FileSystem; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** - * Creates ingest tasks for data source ingest jobs, queueing the tasks in - * priority order for execution by the ingest manager's ingest threads. + * Creates ingest tasks for ingest jobs, queueing the tasks in priority order + * for execution by the ingest manager's ingest threads. */ @ThreadSafe final class IngestTasksScheduler { @@ -54,19 +58,20 @@ final class IngestTasksScheduler { private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName()); @GuardedBy("IngestTasksScheduler.this") private static IngestTasksScheduler instance; - private final IngestTaskTrackingQueue dataSourceIngestThreadQueue; + private final IngestTaskTrackingQueue dataSourceIngestTasksQueue; @GuardedBy("this") - private final TreeSet rootFileTaskQueue; + private final TreeSet topLevelFileIngestTasksQueue; @GuardedBy("this") - private final Deque pendingFileTaskQueue; + private final Deque batchedFileIngestTasksQueue; @GuardedBy("this") - private final Queue streamedTasksQueue; - private final IngestTaskTrackingQueue fileIngestThreadsQueue; + private final Queue streamedFileIngestTasksQueue; + private final IngestTaskTrackingQueue fileIngestTasksQueue; + private final IngestTaskTrackingQueue artifactIngestTasksQueue; /** * Gets the ingest tasks scheduler singleton that creates ingest tasks for - * data source ingest jobs, queueing the tasks in priority order for - * execution by the ingest manager's ingest threads. + * ingest jobs, queueing the tasks in priority order for execution by the + * ingest manager's ingest threads. */ synchronized static IngestTasksScheduler getInstance() { if (IngestTasksScheduler.instance == null) { @@ -76,146 +81,195 @@ final class IngestTasksScheduler { } /** - * Constructs an ingest tasks scheduler that creates ingest tasks for data - * source ingest jobs, queueing the tasks in priority order for execution by - * the ingest manager's ingest threads. + * Constructs an ingest tasks scheduler that creates ingest tasks for ingest + * jobs, queueing the tasks in priority order for execution by the ingest + * manager's ingest threads. */ private IngestTasksScheduler() { - this.dataSourceIngestThreadQueue = new IngestTaskTrackingQueue(); - this.rootFileTaskQueue = new TreeSet<>(new RootDirectoryTaskComparator()); - this.pendingFileTaskQueue = new LinkedList<>(); - this.fileIngestThreadsQueue = new IngestTaskTrackingQueue(); - this.streamedTasksQueue = new LinkedList<>(); + dataSourceIngestTasksQueue = new IngestTaskTrackingQueue(); + topLevelFileIngestTasksQueue = new TreeSet<>(new RootDirectoryTaskComparator()); + batchedFileIngestTasksQueue = new LinkedList<>(); + fileIngestTasksQueue = new IngestTaskTrackingQueue(); + streamedFileIngestTasksQueue = new LinkedList<>(); + artifactIngestTasksQueue = new IngestTaskTrackingQueue(); } /** * Gets the data source level ingest tasks queue. This queue is a blocking - * queue used by the ingest manager's data source level ingest thread. + * queue consumed by the ingest manager's data source level ingest thread. * * @return The queue. */ BlockingIngestTaskQueue getDataSourceIngestTaskQueue() { - return this.dataSourceIngestThreadQueue; + return dataSourceIngestTasksQueue; } /** * Gets the file level ingest tasks queue. This queue is a blocking queue - * used by the ingest manager's file level ingest threads. + * consumed by the ingest manager's file level ingest threads. * * @return The queue. */ BlockingIngestTaskQueue getFileIngestTaskQueue() { - return this.fileIngestThreadsQueue; + return fileIngestTasksQueue; } /** - * Schedules a data source level ingest task and zero to many file level - * ingest tasks for an ingest job pipeline. + * Gets the data artifact ingest tasks queue. This queue is a blocking queue + * consumed by the ingest manager's data artifact ingest thread. * - * @param ingestJobPipeline The ingest job pipeline. + * @return The queue. */ - synchronized void scheduleIngestTasks(IngestJobPipeline ingestJobPipeline) { - if (!ingestJobPipeline.isCancelled()) { - /* - * Scheduling of both the data source ingest task and the initial - * file ingest tasks for an ingestJobPipeline must be an atomic operation. - * Otherwise, the data source task might be completed before the - * file tasks are scheduled, resulting in a potential false positive - * when another thread checks whether or not all the tasks for the - * ingestJobPipeline are completed. - */ - this.scheduleDataSourceIngestTask(ingestJobPipeline); - this.scheduleFileIngestTasks(ingestJobPipeline, Collections.emptyList()); + BlockingIngestTaskQueue getResultIngestTaskQueue() { + return artifactIngestTasksQueue; + } + + /** + * Schedules ingest tasks based on the types of ingest modules that the + * ingest pipeline that will exedute tasks has. Scheduling these tasks + * atomically means that it is valid to call currentTasksAreCompleted() + * immediately after calling this method. Note that the may cause some or + * even all of any file tasks to be discarded. + * + * @param ingestPipeline The ingest pipeline that will execute the scheduled + * tasks. A reference to the pipeline is added to each + * task so that when the task is dequeued by an ingest + * thread the task can pass the target Content of the + * task to the pipeline for processing by the + * pipeline's ingest modules. + */ + synchronized void scheduleIngestTasks(IngestJobPipeline ingestPipeline) { + if (!ingestPipeline.isCancelled()) { + if (ingestPipeline.hasDataSourceIngestModules()) { + scheduleDataSourceIngestTask(ingestPipeline); + } + if (ingestPipeline.hasFileIngestModules()) { + scheduleFileIngestTasks(ingestPipeline, Collections.emptyList()); + } + if (ingestPipeline.hasDataArtifactIngestModules()) { + scheduleDataArtifactIngestTasks(ingestPipeline); + } } } /** - * Schedules a data source level ingest task for an ingest job pipeline. + * Schedules a data source level ingest task for an ingest job. The data + * source is obtained from the ingest pipeline passed in. * - * @param ingestJobPipeline The ingest job pipeline. + * @param ingestPipeline The ingest pipeline that will execute the scheduled + * task. A reference to the pipeline is added to the + * task so that when the task is dequeued by an ingest + * thread the task can pass the target Content of the + * task to the pipeline for processing by the + * pipeline's ingest modules. */ - synchronized void scheduleDataSourceIngestTask(IngestJobPipeline ingestJobPipeline) { - if (!ingestJobPipeline.isCancelled()) { - DataSourceIngestTask task = new DataSourceIngestTask(ingestJobPipeline); + synchronized void scheduleDataSourceIngestTask(IngestJobPipeline ingestPipeline) { + if (!ingestPipeline.isCancelled()) { + DataSourceIngestTask task = new DataSourceIngestTask(ingestPipeline); try { - this.dataSourceIngestThreadQueue.putLast(task); + dataSourceIngestTasksQueue.putLast(task); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (jobId={%d)", ingestJobPipeline.getId()), ex); + IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while blocked adding a task to the data source level ingest task queue (pipelineId={%d)", ingestPipeline.getId()), ex); Thread.currentThread().interrupt(); } } } /** - * Schedules file tasks for either all the files or a given subset of the - * files for an ingest job pipeline. + * Schedules file tasks for either all the files, or a given subset of the + * files, for a data source. The data source is obtained from the ingest + * pipeline passed in. * - * @param ingestJobPipeline The ingest job pipeline. - * @param files A subset of the files for the data source; if empty, then - * file tasks for all files in the data source are scheduled. + * @param ingestPipeline The ingest pipeline that will execute the scheduled + * tasks. A reference to the pipeline is added to each + * task so that when the task is dequeued by an ingest + * thread the task can pass the target Content of the + * task to the pipeline for processing by the + * pipeline's ingest modules. + * @param files A subset of the files from the data source; if + * empty, then all if the files from the data source + * are candidates for scheduling. */ - synchronized void scheduleFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { - if (!ingestJobPipeline.isCancelled()) { + synchronized void scheduleFileIngestTasks(IngestJobPipeline ingestPipeline, Collection files) { + if (!ingestPipeline.isCancelled()) { Collection candidateFiles; if (files.isEmpty()) { - candidateFiles = getTopLevelFiles(ingestJobPipeline.getDataSource()); + candidateFiles = getTopLevelFiles(ingestPipeline.getDataSource()); } else { candidateFiles = files; } for (AbstractFile file : candidateFiles) { - FileIngestTask task = new FileIngestTask(ingestJobPipeline, file); + FileIngestTask task = new FileIngestTask(ingestPipeline, file); if (IngestTasksScheduler.shouldEnqueueFileTask(task)) { - this.rootFileTaskQueue.add(task); + topLevelFileIngestTasksQueue.add(task); } } - refillIngestThreadQueue(); + refillFileIngestTasksQueue(); } } - + /** - * Schedules file tasks for the given list of file IDs. + * Schedules file tasks for a collection of "streamed" files for a streaming + * ingest job. * - * @param ingestJobPipeline The ingest job pipeline. - * @param files A subset of the files for the data source; if empty, then - * file tasks for all files in the data source are scheduled. + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + * @param files A list of file object IDs for the streamed files. */ - synchronized void scheduleStreamedFileIngestTasks(IngestJobPipeline ingestJobPipeline, List fileIds) { - if (!ingestJobPipeline.isCancelled()) { + synchronized void scheduleStreamedFileIngestTasks(IngestJobPipeline ingestPipeline, List fileIds) { + if (!ingestPipeline.isCancelled()) { for (long id : fileIds) { - // Create the file ingest task. Note that we do not do the shouldEnqueueFileTask() - // check here in order to delay loading the AbstractFile object. - FileIngestTask task = new FileIngestTask(ingestJobPipeline, id); - this.streamedTasksQueue.add(task); + /* + * Create the file ingest task. Note that we do not do the + * shouldEnqueueFileTask() check here in order to delay querying + * the case database to construct the AbstractFile object. The + * file filter will be applied before the file task makes it to + * the task queue consumed by the file ingest threads. + */ + FileIngestTask task = new FileIngestTask(ingestPipeline, id); + streamedFileIngestTasksQueue.add(task); } - refillIngestThreadQueue(); + refillFileIngestTasksQueue(); } - } + } /** * Schedules file level ingest tasks for a given set of files for an ingest - * job pipeline by adding them directly to the front of the file tasks - * queue for the ingest manager's file ingest threads. + * job by adding them directly to the front of the file tasks queue consumed + * by the ingest manager's file ingest threads. This method is intended to + * be used to schedule files that are products of ingest module processing, + * e.g., extracted files and carved files. * - * @param ingestJobPipeline The ingestJobPipeline. - * @param files A set of files for the data source. + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + * @param files The files. */ - synchronized void fastTrackFileIngestTasks(IngestJobPipeline ingestJobPipeline, Collection files) { - if (!ingestJobPipeline.isCancelled()) { + synchronized void fastTrackFileIngestTasks(IngestJobPipeline ingestPipeline, Collection files) { + if (!ingestPipeline.isCancelled()) { /* * Put the files directly into the queue for the file ingest * threads, if they pass the file filter for the job. The files are * added to the queue for the ingest threads BEFORE the other queued * tasks because the use case for this method is scheduling new - * carved or derived files from a higher priority task that is - * already in progress. + * carved or derived files from a high priority task that is already + * in progress. */ for (AbstractFile file : files) { - FileIngestTask fileTask = new FileIngestTask(ingestJobPipeline, file); + FileIngestTask fileTask = new FileIngestTask(ingestPipeline, file); if (shouldEnqueueFileTask(fileTask)) { try { - this.fileIngestThreadsQueue.putFirst(fileTask); + fileIngestTasksQueue.putFirst(fileTask); } catch (InterruptedException ex) { - IngestTasksScheduler.logger.log(Level.INFO, String.format("Ingest tasks scheduler interrupted while scheduling file level ingest tasks (jobId={%d)", ingestJobPipeline.getId()), ex); + DataSource dataSource = ingestPipeline.getDataSource(); + logger.log(Level.WARNING, String.format("Interrupted while enqueuing file tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS Thread.currentThread().interrupt(); return; } @@ -224,6 +278,62 @@ final class IngestTasksScheduler { } } + /** + * Schedules data artifact ingest tasks for any data artifacts that have + * already been added to the case database for a data source. The data + * source is obtained from the ingest pipeline passed in. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + */ + synchronized void scheduleDataArtifactIngestTasks(IngestJobPipeline ingestPipeline) { + if (!ingestPipeline.isCancelled()) { + Blackboard blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); + try { + List artifacts = blackboard.getDataArtifacts(ingestPipeline.getDataSource().getId(), null); + scheduleDataArtifactIngestTasks(ingestPipeline, artifacts); + } catch (TskCoreException ex) { + DataSource dataSource = ingestPipeline.getDataSource(); + logger.log(Level.SEVERE, String.format("Failed to retrieve data artifacts for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS + } + } + } + + /** + * Schedules data artifact ingest tasks for an ingest job. This method is + * intended to be used to schedule artifacts that are products of ingest + * module processing. + * + * @param ingestPipeline The ingest pipeline for the job. A reference to the + * pipeline is added to each task so that when the + * task is dequeued by an ingest thread and the task's + * execute() method is called, execute() can pass the + * target Content of the task to the pipeline for + * processing by the pipeline's ingest modules. + * @param artifacts A subset of the data artifacts from the data + * source; if empty, then all of the data artifacts + * from the data source will be scheduled. + */ + synchronized void scheduleDataArtifactIngestTasks(IngestJobPipeline ingestPipeline, List artifacts) { + if (!ingestPipeline.isCancelled()) { + for (DataArtifact artifact : artifacts) { + DataArtifactIngestTask task = new DataArtifactIngestTask(ingestPipeline, artifact); + try { + this.artifactIngestTasksQueue.putLast(task); + } catch (InterruptedException ex) { + DataSource dataSource = ingestPipeline.getDataSource(); + logger.log(Level.WARNING, String.format("Interrupted while enqueuing data artifact tasks for %s (data source object ID = %d)", dataSource.getName(), dataSource.getId()), ex); //NON-NLS + Thread.currentThread().interrupt(); + break; + } + } + } + } + /** * Allows an ingest thread to notify this ingest task scheduler that a data * source level task has been completed. @@ -231,7 +341,7 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(DataSourceIngestTask task) { - this.dataSourceIngestThreadQueue.taskCompleted(task); + dataSourceIngestTasksQueue.taskCompleted(task); } /** @@ -241,46 +351,67 @@ final class IngestTasksScheduler { * @param task The completed task. */ synchronized void notifyTaskCompleted(FileIngestTask task) { - this.fileIngestThreadsQueue.taskCompleted(task); - refillIngestThreadQueue(); + fileIngestTasksQueue.taskCompleted(task); + refillFileIngestTasksQueue(); + } + + /** + * Allows an ingest thread to notify this ingest task scheduler that a data + * artifact ingest task has been completed. + * + * @param task The completed task. + */ + synchronized void notifyTaskCompleted(DataArtifactIngestTask task) { + artifactIngestTasksQueue.taskCompleted(task); } /** * Queries the task scheduler to determine whether or not all of the ingest - * tasks for an ingest job pipeline have been completed. + * tasks for an ingest job have been completed. * - * @param ingestJobPipeline The ingestJobPipeline. + * @param ingestPipeline The ingest pipeline for the job. * * @return True or false. */ - synchronized boolean currentTasksAreCompleted(IngestJobPipeline ingestJobPipeline) { - long jobId = ingestJobPipeline.getId(); - - return !(this.dataSourceIngestThreadQueue.hasTasksForJob(jobId) - || hasTasksForJob(this.rootFileTaskQueue, jobId) - || hasTasksForJob(this.pendingFileTaskQueue, jobId) - || hasTasksForJob(this.streamedTasksQueue, jobId) - || this.fileIngestThreadsQueue.hasTasksForJob(jobId)); + synchronized boolean currentTasksAreCompleted(IngestJobPipeline ingestPipeline) { + long pipelineId = ingestPipeline.getId(); + return !(dataSourceIngestTasksQueue.hasTasksForJob(pipelineId) + || hasTasksForJob(topLevelFileIngestTasksQueue, pipelineId) + || hasTasksForJob(batchedFileIngestTasksQueue, pipelineId) + || hasTasksForJob(streamedFileIngestTasksQueue, pipelineId) + || fileIngestTasksQueue.hasTasksForJob(pipelineId) + || artifactIngestTasksQueue.hasTasksForJob(pipelineId)); } /** - * Clears the "upstream" task scheduling queues for an ingest pipeline, - * but does nothing about tasks that have already been moved into the - * queue that is consumed by the file ingest threads. + * Cancels the pending file ingest tasks for an ingest job, where the + * pending tasks are the file ingest tasks that are in the upstream + * scheduling queues (batch and streaming) that feed into the queue consumed + * by the ingest manager's file ingest threads. * - * @param ingestJobPipeline The ingestJobPipeline. + * Note that the "normal" way to cancel an ingest job is to mark the job as + * cancelled, which causes the execute() methods of the ingest tasks for the + * job to return immediately when called, leading to flushing all of the + * tasks for the job out of the ingest task queues by the ingest threads and + * an orderly progression through IngestTaskTrackingQueue bookkeeping and + * the ingest job stages to early job completion. However, this method is a + * cancellation speed booster. For example, it eliminates the creation of + * what could be a large number of child tasks for both the top level files + * in the batch root file tasks queue and any directories in the batch root + * children file tasks queue. + * + * @param ingestJobPipeline The ingest pipeline for the job. */ - synchronized void cancelPendingTasksForIngestJob(IngestJobPipeline ingestJobPipeline) { + synchronized void cancelPendingFileTasksForIngestJob(IngestJobPipeline ingestJobPipeline) { long jobId = ingestJobPipeline.getId(); - IngestTasksScheduler.removeTasksForJob(rootFileTaskQueue, jobId); - IngestTasksScheduler.removeTasksForJob(pendingFileTaskQueue, jobId); - IngestTasksScheduler.removeTasksForJob(streamedTasksQueue, jobId); + removeTasksForJob(topLevelFileIngestTasksQueue, jobId); + removeTasksForJob(batchedFileIngestTasksQueue, jobId); + removeTasksForJob(streamedFileIngestTasksQueue, jobId); } /** - * Gets the top level files such as file system root directories, layout - * files and virtual directories for a data source. Used to create file - * tasks to put into the root directories queue. + * Gets the top level files for a data source, such as file system root + * directories, layout files, and virtual directories. * * @param dataSource The data source. * @@ -311,20 +442,23 @@ final class IngestTasksScheduler { } } } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS + logger.log(Level.SEVERE, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS } } } return topLevelFiles; } - + /** - * Schedules file ingest tasks for the ingest manager's file ingest threads. - * Files from streaming ingest will be prioritized. + * Refills the file ingest tasks queue consumed by the ingest manager's file + * ingest threads with tasks from the upstream file task scheduling queues + * (streamed and batch). Files from the streamed file ingest tasks queue are + * prioritized. Applies the file filter for the ingest job and attempts to + * move as many tasks as there are ingest threads. */ - synchronized private void refillIngestThreadQueue() { + synchronized private void refillFileIngestTasksQueue() { try { - takeFromStreamingTaskQueue(); + takeFromStreamingFileTasksQueue(); takeFromBatchTasksQueues(); } catch (InterruptedException ex) { IngestTasksScheduler.logger.log(Level.INFO, "Ingest tasks scheduler interrupted while blocked adding a task to the file level ingest task queue", ex); @@ -333,27 +467,21 @@ final class IngestTasksScheduler { } /** - * Move tasks from the streamedTasksQueue into the fileIngestThreadsQueue. - * Will attempt to move as many tasks as there are ingest threads. + * Moves tasks from the upstream streamed file ingest tasks queue into the + * file ingest tasks queue consumed by the ingest manager's file ingest + * threads. Applies the file filter for the ingest job and attempts to move + * as many tasks as there are ingest threads. */ - synchronized private void takeFromStreamingTaskQueue() throws InterruptedException { - /* - * Schedule files from the streamedTasksQueue - */ - while (fileIngestThreadsQueue.isEmpty()) { - /* - * We will attempt to schedule as many tasks as there are ingest - * queues. - */ + synchronized private void takeFromStreamingFileTasksQueue() throws InterruptedException { + while (fileIngestTasksQueue.isEmpty()) { int taskCount = 0; while (taskCount < IngestManager.getInstance().getNumberOfFileIngestThreads()) { - final FileIngestTask streamingTask = streamedTasksQueue.poll(); + final FileIngestTask streamingTask = streamedFileIngestTasksQueue.poll(); if (streamingTask == null) { return; // No streaming tasks are queued right now } - if (shouldEnqueueFileTask(streamingTask)) { - fileIngestThreadsQueue.putLast(streamingTask); + fileIngestTasksQueue.putLast(streamingTask); taskCount++; } } @@ -361,104 +489,91 @@ final class IngestTasksScheduler { } /** - * Schedules file ingest tasks for the ingest manager's file ingest threads - * by "shuffling" them through a sequence of three queues that allows for - * the interleaving of tasks from different data source ingest jobs based on - * priority, while limiting the number of queued tasks by only expanding - * directories one at a time. The sequence of queues is: + * Moves tasks from the upstream batched file ingest task queues into the + * file ingest tasks queue consumed by the ingest manager's file ingest + * threads. A sequence of two upstream queues is used to interleave tasks + * from different ingest jobs based on priority. Applies the file filter for + * the ingest job and attempts to move as many tasks as there are ingest + * threads. * - * 1. The root file tasks priority queue, which contains file tasks for the - * root objects of the data sources that are being analyzed. For example, - * the root tasks for a disk image data source are typically the tasks for - * the contents of the root directories of the file systems. This queue is a - * priority queue that attempts to ensure that user content is analyzed - * before general file system content. It feeds into the pending tasks - * queue. + * The upstream batched file task queues are: * - * 2. The pending file tasks queue, which contains root file tasks shuffled - * out of the root tasks queue, plus tasks for files with children - * discovered in the descent from the root tasks to the final leaf tasks in - * the content trees that are being analyzed for the data source ingest - * jobs. This queue is a FIFO queue that attempts to throttle the total - * number of file tasks by deferring queueing tasks for the children of - * files until the queue for the file ingest threads is emptied. It feeds - * into the file tasks queue for the ingest manager's file ingest threads. + * 1. The top level file tasks queue, which contains file tasks for the root + * objects of data sources. For example, the top level file tasks for a disk + * image data source are typically the tasks for the contents of the root + * directories of the file systems. This queue is a priority queue that + * attempts to ensure that user content is analyzed before general file + * system content. It feeds into the batched file ingest tasks queue. * - * 3. The file tasks queue for the ingest manager's file ingest threads. - * This queue is a blocking deque that is FIFO during a shuffle to maintain - * task prioritization, but LIFO when adding derived files to it directly - * during ingest. The reason for the LIFO additions is to give priority to - * files derived from prioritized files. + * 2. The batch file tasks queue, which contains top level file tasks moved + * in from the top level file tasks queue, plus tasks for child files in the + * descent from the root tasks to the final leaf tasks in the content trees + * that are being analyzed for any given data source. This queue is a FIFO + * queue that attempts to throttle the total number of file tasks by + * deferring queueing of tasks for the children of files until the queue for + * the file ingest threads is emptied. */ synchronized private void takeFromBatchTasksQueues() throws InterruptedException { - - while (this.fileIngestThreadsQueue.isEmpty()) { + + while (fileIngestTasksQueue.isEmpty()) { /* - * If the pending file task queue is empty, move the highest - * priority root file task, if there is one, into it. + * If the batched file task queue is empty, move the highest + * priority top level file task into it. */ - if (this.pendingFileTaskQueue.isEmpty()) { - final FileIngestTask rootTask = this.rootFileTaskQueue.pollFirst(); - if (rootTask != null) { - this.pendingFileTaskQueue.addLast(rootTask); + if (batchedFileIngestTasksQueue.isEmpty()) { + final FileIngestTask topLevelTask = topLevelFileIngestTasksQueue.pollFirst(); + if (topLevelTask != null) { + batchedFileIngestTasksQueue.addLast(topLevelTask); } } /* - * Try to move the next task from the pending task queue into the - * queue for the file ingest threads, if it passes the filter for - * the job. + * Try to move the next task from the batched file tasks queue into + * the queue for the file ingest threads. */ - final FileIngestTask pendingTask = this.pendingFileTaskQueue.pollFirst(); - if (pendingTask == null) { + final FileIngestTask nextTask = batchedFileIngestTasksQueue.pollFirst(); + if (nextTask == null) { return; } - if (shouldEnqueueFileTask(pendingTask)) { - /* - * The task is added to the queue for the ingest threads - * AFTER the higher priority tasks that preceded it. - */ - this.fileIngestThreadsQueue.putLast(pendingTask); + if (shouldEnqueueFileTask(nextTask)) { + fileIngestTasksQueue.putLast(nextTask); } /* * If the task that was just queued for the file ingest threads has - * children, try to queue tasks for the children. Each child task - * will go into either the directory queue if it has children of its - * own, or into the queue for the file ingest threads, if it passes - * the filter for the job. + * children, queue tasks for the children as well. */ AbstractFile file = null; try { - file = pendingTask.getFile(); + file = nextTask.getFile(); for (Content child : file.getChildren()) { if (child instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) child; - FileIngestTask childTask = new FileIngestTask(pendingTask.getIngestJobPipeline(), childFile); + FileIngestTask childTask = new FileIngestTask(nextTask.getIngestJobPipeline(), childFile); if (childFile.hasChildren()) { - this.pendingFileTaskQueue.add(childTask); + batchedFileIngestTasksQueue.add(childTask); } else if (shouldEnqueueFileTask(childTask)) { - this.fileIngestThreadsQueue.putLast(childTask); + fileIngestTasksQueue.putLast(childTask); } } } } catch (TskCoreException ex) { if (file != null) { - logger.log(Level.SEVERE, String.format("Error getting the children of %s (objId=%d)", file.getName(), file.getId()), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error getting the children of %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS } else { - // In practice, the task would have already returned false from the call to shouldEnqueueFileTask() - logger.log(Level.SEVERE, "Error loading file with object ID {0}", pendingTask.getFileId()); + logger.log(Level.SEVERE, "Error loading file with object ID = {0}", nextTask.getFileId()); //NON-NLS } } } } /** - * Examines the file associated with a file ingest task to determine whether - * or not the file should be processed and therefore whether or not the task - * should be enqueued. + * Evaluates the file for a file ingest task to determine whether or not the + * file should be processed and therefore whether or not the task should be + * enqueued. The evaluation includes applying the file filter for the task's + * parent ingest job. * - * @param task The task to be scrutinized. + * @param task The task. * * @return True or false. */ @@ -544,11 +659,10 @@ final class IngestTasksScheduler { } /** - * Check whether or not a file should be carved for a data source ingest - * ingest job. - * + * Checks whether or not a file should be carved for an ingest job. + * * @param task The file ingest task for the file. - * + * * @return True or false. */ private static boolean shouldBeCarved(final FileIngestTask task) { @@ -561,12 +675,12 @@ final class IngestTasksScheduler { } /** - * Checks whether or not a file is accepted (passes) the file filter for a data - * source ingest job. + * Checks whether or not a file is accepted (passes) the file filter for an + * ingest job. * * @param task The file ingest task for the file. - * - * @return True or false. + * + * @return True or false. */ private static boolean fileAcceptedByFilter(final FileIngestTask task) { try { @@ -579,16 +693,16 @@ final class IngestTasksScheduler { /** * Checks whether or not a collection of ingest tasks includes a task for a - * given data source ingest job. + * given ingest job. * - * @param tasks The tasks. - * @param jobId The data source ingest job id. + * @param tasks The tasks. + * @param pipelineId The ID of the ingest pipeline for the job. * * @return True if there are no tasks for the job, false otherwise. */ - synchronized private static boolean hasTasksForJob(Collection tasks, long jobId) { + synchronized private static boolean hasTasksForJob(Collection tasks, long pipelineId) { for (IngestTask task : tasks) { - if (task.getIngestJobPipeline().getId() == jobId) { + if (task.getIngestJobPipeline().getId() == pipelineId) { return true; } } @@ -596,34 +710,35 @@ final class IngestTasksScheduler { } /** - * Removes all of the ingest tasks associated with a data source ingest job - * from a tasks collection. + * Removes all of the ingest tasks associated with an ingest job from a + * collection of tasks. * - * @param tasks The collection from which to remove the tasks. - * @param jobId The data source ingest job id. + * @param tasks The tasks. + * @param pipelineId The ID of the ingest pipeline for the job. */ - private static void removeTasksForJob(Collection tasks, long jobId) { + private static void removeTasksForJob(Collection tasks, long pipelineId) { Iterator iterator = tasks.iterator(); while (iterator.hasNext()) { IngestTask task = iterator.next(); - if (task.getIngestJobPipeline().getId() == jobId) { + if (task.getIngestJobPipeline().getId() == pipelineId) { iterator.remove(); } } } /** - * Counts the number of ingest tasks in a tasks collection for a given job. + * Counts the number of ingest tasks in a collection of tasks for a given + * ingest job. * - * @param queue The queue for which to count tasks. - * @param jobId The id of the job for which the tasks are to be counted. + * @param tasks The tasks. + * @param pipelineId The ID of the ingest pipeline for the job. * * @return The count. */ - private static int countTasksForJob(Collection queue, long jobId) { + private static int countTasksForJob(Collection tasks, long pipelineId) { int count = 0; - for (IngestTask task : queue) { - if (task.getIngestJobPipeline().getId() == jobId) { + for (IngestTask task : tasks) { + if (task.getIngestJobPipeline().getId() == pipelineId) { count++; } } @@ -639,12 +754,13 @@ final class IngestTasksScheduler { * @return */ synchronized IngestJobTasksSnapshot getTasksSnapshotForJob(long jobId) { - return new IngestJobTasksSnapshot(jobId, this.dataSourceIngestThreadQueue.countQueuedTasksForJob(jobId), - countTasksForJob(this.rootFileTaskQueue, jobId), - countTasksForJob(this.pendingFileTaskQueue, jobId), - this.fileIngestThreadsQueue.countQueuedTasksForJob(jobId), - this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + this.fileIngestThreadsQueue.countRunningTasksForJob(jobId), - countTasksForJob(this.streamedTasksQueue, jobId)); + return new IngestJobTasksSnapshot(jobId, dataSourceIngestTasksQueue.countQueuedTasksForJob(jobId), + countTasksForJob(topLevelFileIngestTasksQueue, jobId), + countTasksForJob(batchedFileIngestTasksQueue, jobId), + fileIngestTasksQueue.countQueuedTasksForJob(jobId), + dataSourceIngestTasksQueue.countRunningTasksForJob(jobId) + fileIngestTasksQueue.countRunningTasksForJob(jobId) + artifactIngestTasksQueue.countRunningTasksForJob(jobId), + countTasksForJob(streamedFileIngestTasksQueue, jobId), + artifactIngestTasksQueue.countQueuedTasksForJob(jobId)); } /** @@ -664,23 +780,23 @@ final class IngestTasksScheduler { } catch (TskCoreException ex) { // Do nothing - the exception has been logged elsewhere } - + try { file2 = q2.getFile(); } catch (TskCoreException ex) { // Do nothing - the exception has been logged elsewhere } - + if (file1 == null) { if (file2 == null) { return (int) (q2.getFileId() - q1.getFileId()); } else { return 1; } - } else if (file2 == null) { + } else if (file2 == null) { return -1; } - + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(file1); AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(file2); if (p1 == p2) { @@ -883,7 +999,7 @@ final class IngestTasksScheduler { /** * Handles the completion of an ingest task by removing it from the - * running tasks list. + * tasks in progress list. * * @param task The completed task. */ @@ -895,43 +1011,41 @@ final class IngestTasksScheduler { /** * Checks whether there are any ingest tasks are queued and/or running - * for a given data source ingest job. + * for a given ingest job. * - * @param jobId The id of the data source ingest job. + * @param pipelineId The ID of the ingest pipeline for the job. * * @return */ - boolean hasTasksForJob(long jobId) { + boolean hasTasksForJob(long pipelineId) { synchronized (this) { - return IngestTasksScheduler.hasTasksForJob(this.queuedTasks, jobId) || IngestTasksScheduler.hasTasksForJob(this.tasksInProgress, jobId); + return IngestTasksScheduler.hasTasksForJob(queuedTasks, pipelineId) || IngestTasksScheduler.hasTasksForJob(tasksInProgress, pipelineId); } } /** - * Gets a count of the queued ingest tasks for a given data source - * ingest job. + * Gets a count of the queued ingest tasks for a given ingest job. * - * @param jobId + * @param pipelineId The ID of the ingest pipeline for the job. * * @return */ - int countQueuedTasksForJob(long jobId) { + int countQueuedTasksForJob(long pipelineId) { synchronized (this) { - return IngestTasksScheduler.countTasksForJob(this.queuedTasks, jobId); + return IngestTasksScheduler.countTasksForJob(queuedTasks, pipelineId); } } /** - * Gets a count of the running ingest tasks for a given data source - * ingest job. + * Gets a count of the running ingest tasks for a given ingest job. * - * @param jobId + * @param pipelineId The ID of the ingest pipeline for the job. * * @return */ - int countRunningTasksForJob(long jobId) { + int countRunningTasksForJob(long pipelineId) { synchronized (this) { - return IngestTasksScheduler.countTasksForJob(this.tasksInProgress, jobId); + return IngestTasksScheduler.countTasksForJob(tasksInProgress, pipelineId); } } @@ -950,14 +1064,24 @@ final class IngestTasksScheduler { private final long fileQueueSize; private final long runningListSize; private final long streamingQueueSize; + private final long artifactsQueueSize; /** + * RJCTODO + * * Constructs a snapshot of ingest tasks data for an ingest job. * - * @param jobId The identifier associated with the job. + * @param jobId The identifier associated with the job. + * @param dsQueueSize + * @param rootQueueSize + * @param dirQueueSize + * @param fileQueueSize + * @param runningListSize + * @param streamingQueueSize + * @param artifactsQueueSize */ - IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, - long runningListSize, long streamingQueueSize) { + IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, + long runningListSize, long streamingQueueSize, long artifactsQueueSize) { this.jobId = jobId; this.dsQueueSize = dsQueueSize; this.rootQueueSize = rootQueueSize; @@ -965,6 +1089,7 @@ final class IngestTasksScheduler { this.fileQueueSize = fileQueueSize; this.runningListSize = runningListSize; this.streamingQueueSize = streamingQueueSize; + this.artifactsQueueSize = artifactsQueueSize; } /** @@ -1000,7 +1125,7 @@ final class IngestTasksScheduler { long getFileQueueSize() { return fileQueueSize; } - + long getStreamingQueueSize() { return streamingQueueSize; } @@ -1012,6 +1137,10 @@ final class IngestTasksScheduler { long getRunningListSize() { return runningListSize; } + + long getArtifactsQueueSize() { + return artifactsQueueSize; + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java index dc7eceaad8..9a66992900 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/Snapshot.java @@ -193,6 +193,13 @@ public final class Snapshot implements Serializable { return this.tasksSnapshot.getRunningListSize(); } + long getArtifactTasksQueueSize() { + if (tasksSnapshot == null) { + return 0; + } + return tasksSnapshot.getArtifactsQueueSize(); + } + boolean isCancelled() { return this.jobCancelled; } diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED index e8411caa04..efee783e8f 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED @@ -12,7 +12,12 @@ ExtractArchiveWithPasswordAction.progress.text=Unpacking contents of archive: {0 ExtractArchiveWithPasswordAction.prompt.text=Enter Password ExtractArchiveWithPasswordAction.prompt.title=Enter Password OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Embedded File Extraction Ingest Module\n\nThe Embedded File Extraction Ingest Module processes document files (such as doc, docx, ppt, pptx, xls, xlsx) and archive files (such as zip and others archive types supported by the 7zip extractor).\nContents of these files are extracted and the derived files are added back to the current ingest to be processed by the configured ingest modules.\nIf the derived file happens to be an archive file, it will be re-processed by the 7zip extractor - the extractor will process archive files N-levels deep.\n\nThe extracted files are navigable in the directory tree.\n\nThe module is supported on Windows, Linux and Mac operating systems. +OpenIDE-Module-Long-Description=\ + Embedded File Extraction Ingest Module\n\nThe Embedded File Extraction Ingest Module processes document files (such as doc, docx, ppt, pptx, xls, xlsx) and archive files (such as zip and others archive types supported by the 7zip extractor).\n\ + Contents of these files are extracted and the derived files are added back to the current ingest to be processed by the configured ingest modules.\n\ + If the derived file happens to be an archive file, it will be re-processed by the 7zip extractor - the extractor will process archive files N-levels deep.\n\n\ + The extracted files are navigable in the directory tree.\n\n\ + The module is supported on Windows, Linux and Mac operating systems. OpenIDE-Module-Name=Embedded File Extraction OpenIDE-Module-Short-Description=Embedded File Extraction Ingest Module EmbeddedFileExtractorIngestModule.SevenZipContentReadStream.seek.exception.invalidOrigin=Invalid seek origin: {0} diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED index 5063bd55fa..cfaadf1635 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/Bundle.properties-MERGED @@ -36,27 +36,27 @@ FileExtMismatchSettingsPanel.jLabel1.text=File Types: FileExtMismatchSettingsPanel.newExtButton.text=New Extension FileExtMismatchSettingsPanel.newMimePrompt.message=Add a new MIME file type: FileExtMismatchSettingsPanel.newMimePrompt.title=New MIME -FileExtMismatchSettingsPanel.newMimePrompt.emptyMime.message=MIME type text is empty! +FileExtMismatchSettingsPanel.newMimePrompt.emptyMime.message=MIME type text is empty\! FileExtMismatchSettingsPanel.newMimePrompt.emptyMime.title=Empty type -FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotSupported.message=MIME type not supported! +FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotSupported.message=MIME type not supported\! FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotSupported.title=Type not supported -FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeExists.message=MIME type already exists! +FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeExists.message=MIME type already exists\! FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeExists.title=Type already exists FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotDetectable.message=MIME type is not detectable by this module. FileExtMismatchSettingsPanel.newMimePrompt.mimeTypeNotDetectable.title=Type not detectable -FileExtMismatchSettingsPanel.removeTypeButton.noneSelected.message=No MIME type selected! +FileExtMismatchSettingsPanel.removeTypeButton.noneSelected.message=No MIME type selected\! FileExtMismatchSettingsPanel.removeTypeButton.noneSelected.title=No type selected FileExtMismatchSettingsPanel.newExtPrompt.message=Add an allowed extension: FileExtMismatchSettingsPanel.newExtPrompt.title=New allowed extension -FileExtMismatchSettingsPanel.newExtPrompt.empty.message=Extension text is empty! +FileExtMismatchSettingsPanel.newExtPrompt.empty.message=Extension text is empty\! FileExtMismatchSettingsPanel.newExtPrompt.empty.title=Extension text empty -FileExtMismatchSettingsPanel.newExtPrompt.noMimeType.message=No MIME type selected! +FileExtMismatchSettingsPanel.newExtPrompt.noMimeType.message=No MIME type selected\! FileExtMismatchSettingsPanel.newExtPrompt.noMimeType.title=No MIME type selected -FileExtMismatchSettingsPanel.newExtPrompt.extExists.message=Extension already exists! +FileExtMismatchSettingsPanel.newExtPrompt.extExists.message=Extension already exists\! FileExtMismatchSettingsPanel.newExtPrompt.extExists.title=Extension already exists -FileExtMismatchSettingsPanel.removeExtButton.noneSelected.message=No extension selected! +FileExtMismatchSettingsPanel.removeExtButton.noneSelected.message=No extension selected\! FileExtMismatchSettingsPanel.removeExtButton.noneSelected.title=No extension selected -FileExtMismatchSettingsPanel.removeExtButton.noMimeTypeSelected.message=No MIME type selected! +FileExtMismatchSettingsPanel.removeExtButton.noMimeTypeSelected.message=No MIME type selected\! FileExtMismatchSettingsPanel.removeExtButton.noMimeTypeSelected.title=No MIME type selected FileExtMismatchSettingsPanel.removeTypeButton.toolTipText= FileExtMismatchModuleSettingsPanel.checkAllRadioButton.text=Check all file types diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED index 8dbb55e35f..dd5aa258cc 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties-MERGED @@ -61,7 +61,10 @@ ImportCentralRepoDbProgressDialog.errorParsingFile.message=Error parsing hash se ImportCentralRepoDbProgressDialog.linesProcessed.message=\ hashes processed ImportCentralRepoDbProgressDialog.title.text=Central Repository Import Progress OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Hash Set ingest module. \n\nThe ingest module analyzes files in the disk image and marks them as "known" (based on NSRL hashset lookup for "known" files) and "bad / interesting" (based on one or more hash sets supplied by the user).\n\nThe module also contains additional non-ingest tools that are integrated in the GUI, such as file lookup by hash and hash set configuration. +OpenIDE-Module-Long-Description=\ + Hash Set ingest module. \n\n\ + The ingest module analyzes files in the disk image and marks them as "known" (based on NSRL hashset lookup for "known" files) and "bad / interesting" (based on one or more hash sets supplied by the user).\n\n\ + The module also contains additional non-ingest tools that are integrated in the GUI, such as file lookup by hash and hash set configuration. OpenIDE-Module-Name=HashDatabases OptionsCategory_Name_HashDatabase=Hash Sets OptionsCategory_Keywords_HashDatabase=Hash Sets @@ -188,7 +191,10 @@ HashDbSearchThread.name.searching=Searching HashDbSearchThread.noMoreFilesWithMD5Msg=No other files with the same MD5 hash were found. ModalNoButtons.indexingDbsTitle=Indexing hash sets ModalNoButtons.indexingDbTitle=Indexing hash set -ModalNoButtons.exitHashDbIndexingMsg=You are about to exit out of indexing your hash sets. \nThe generated index will be left unusable. If you choose to continue,\nplease delete the corresponding -md5.idx file in the hash folder.\nExit indexing? +ModalNoButtons.exitHashDbIndexingMsg=You are about to exit out of indexing your hash sets. \n\ +The generated index will be left unusable. If you choose to continue,\n\ + please delete the corresponding -md5.idx file in the hash folder.\n\ + Exit indexing? ModalNoButtons.dlgTitle.unfinishedIndexing=Unfinished Indexing ModalNoButtons.indexThis.currentlyIndexing1Db=Currently indexing 1 hash set ModalNoButtons.indexThese.currentlyIndexing1OfNDbs=Currently indexing 1 of {0} diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED index cccbcc1b57..6fb258f014 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties-MERGED @@ -2,6 +2,7 @@ FilesIdentifierIngestJobSettingsPanel.getError=Error getting interesting files s FilesIdentifierIngestJobSettingsPanel.updateError=Error updating interesting files sets settings file. FilesIdentifierIngestModule.getFilesError=Error getting interesting files sets from file. FilesIdentifierIngestModule.indexError.message=Failed to index interesting file hit artifact for keyword search. +# {0} - daysIncluded FilesSet.rule.dateRule.toString=(modified within {0} day(s)) FilesSetDefsPanel.bytes=Bytes FilesSetDefsPanel.cancelImportMsg=Cancel import @@ -121,8 +122,8 @@ FilesSetRulePanel.nameTextField.text= FilesSetRulePanel.ruleNameLabel.text=Rule Name (Optional): FilesSetRulePanel.messages.emptyNameCondition=You must specify a name pattern for this rule. FilesSetRulePanel.messages.invalidNameRegex=The name regular expression is not valid:\n\n{0} -FilesSetRulePanel.messages.invalidCharInName=The name cannot contain \\, /, :, *, ?, ", <, or > unless it is a regular expression. -FilesSetRulePanel.messages.invalidCharInPath=The path cannot contain \\, :, *, ?, ", <, or > unless it is a regular expression. +FilesSetRulePanel.messages.invalidCharInName=The name cannot contain \\, /, :, *, ?, \", <, or > unless it is a regular expression. +FilesSetRulePanel.messages.invalidCharInPath=The path cannot contain \\, :, *, ?, \", <, or > unless it is a regular expression. FilesSetRulePanel.messages.invalidPathRegex=The path regular expression is not valid:\n\n{0} FilesSetDefsPanel.doFileSetsDialog.duplicateRuleSet.text=Rule set with name {0} already exists. FilesSetRulePanel.pathSeparatorInfoLabel.text=Folder must be in parent path. Use '/' to give consecutive names diff --git a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED index f5dd54dc50..1d07988e4c 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/photoreccarver/Bundle.properties-MERGED @@ -24,7 +24,7 @@ PhotoRecIngestModule.complete.totalParsetime=Total Parsing Time: PhotoRecIngestModule.complete.photoRecResults=PhotoRec Results PhotoRecIngestModule.NotEnoughDiskSpace.detail.msg=PhotoRec error processing {0} with {1} Not enough space on primary disk to save unallocated space. PhotoRecIngestModule.cancelledByUser=PhotoRec cancelled by user. -PhotoRecIngestModule.error.exitValue=PhotoRec carver returned error exit value = {0} when scanning {1} +PhotoRecIngestModule.error.exitValue=PhotoRec carver returned error exit value \= {0} when scanning {1} PhotoRecIngestModule.error.msg=Error processing {0} with PhotoRec carver. PhotoRecIngestModule.complete.numberOfErrors=Number of Errors while Carving: PhotoRecCarverIngestJobSettingsPanel.detectionSettingsLabel.text=PhotoRec Settings diff --git a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/Bundle.properties-MERGED index 569370ec6e..b8d6f6e434 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/Bundle.properties-MERGED @@ -1,2 +1 @@ -ExifProcessor.indexError.message=Failed to post EXIF Metadata artifact(s). ExifProcessor.userContent.description=EXIF metadata data exists for this file. diff --git a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java index 99a1920818..ea7dea7bf5 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2021 Basis Technology Corp. + * Copyright 2020-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,6 +34,7 @@ import java.util.Collection; import java.util.Date; import java.util.Set; import java.util.HashSet; +import java.util.List; import java.util.TimeZone; import java.util.logging.Level; import org.apache.commons.lang3.StringUtils; @@ -44,7 +45,6 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.modules.pictureanalyzer.PictureAnalyzerIngestModuleFactory; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -68,10 +68,10 @@ import org.sleuthkit.datamodel.Score; public class EXIFProcessor implements PictureProcessor { private static final Logger logger = Logger.getLogger(EXIFProcessor.class.getName()); + private static final BlackboardArtifact.Type EXIF_METADATA = new BlackboardArtifact.Type(TSK_METADATA_EXIF); @Override @NbBundle.Messages({ - "ExifProcessor.indexError.message=Failed to post EXIF Metadata artifact(s).", "ExifProcessor.userContent.description=EXIF metadata data exists for this file." }) public void process(IngestJobContext context, AbstractFile file) { @@ -148,37 +148,37 @@ public class EXIFProcessor implements PictureProcessor { final Blackboard blackboard = Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard(); if (!attributes.isEmpty() && !blackboard.artifactExists(file, TSK_METADATA_EXIF, attributes)) { - + List artifacts = new ArrayList<>(); final BlackboardArtifact exifArtifact = (file.newAnalysisResult( BlackboardArtifact.Type.TSK_METADATA_EXIF, Score.SCORE_NONE, null, null, null, attributes)).getAnalysisResult(); - + artifacts.add(exifArtifact); + final BlackboardArtifact userSuspectedArtifact = file.newAnalysisResult( - BlackboardArtifact.Type.TSK_USER_CONTENT_SUSPECTED, Score.SCORE_UNKNOWN, null, null, null, - Arrays.asList(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, Bundle.ExifProcessor_userContent_description()))) + BlackboardArtifact.Type.TSK_USER_CONTENT_SUSPECTED, + Score.SCORE_UNKNOWN, + null, null, null, + Arrays.asList(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, + MODULE_NAME, + Bundle.ExifProcessor_userContent_description()))) .getAnalysisResult(); - + artifacts.add(userSuspectedArtifact); + try { - // index the artifact for keyword search - blackboard.postArtifact(exifArtifact, MODULE_NAME); - blackboard.postArtifact(userSuspectedArtifact, MODULE_NAME); + blackboard.postArtifacts(artifacts, MODULE_NAME); } catch (Blackboard.BlackboardException ex) { - logger.log(Level.SEVERE, "Unable to index blackboard artifact " + exifArtifact.getArtifactID(), ex); //NON-NLS - MessageNotifyUtil.Notify.error( - Bundle.ExifProcessor_indexError_message(), exifArtifact.getDisplayName()); + logger.log(Level.SEVERE, String.format("Error posting TSK_METADATA_EXIF and TSK_USER_CONTENT_SUSPECTED artifacts for %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS } } } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Failed to create blackboard artifact for " //NON-NLS - + "exif metadata ({0}).", ex.getLocalizedMessage()); //NON-NLS - } catch (IOException | ImageProcessingException unused) { - // In this case the stack trace is not needed in the log. - logger.log(Level.WARNING, String.format("Error parsing " //NON-NLS - + "image file '%s/%s' (id=%d).", file.getParentPath(), file.getName(), file.getId())); //NON-NLS + logger.log(Level.SEVERE, String.format("Error creating TSK_METADATA_EXIF and TSK_USER_CONTENT_SUSPECTED artifacts for %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS + } catch (IOException | ImageProcessingException ex) { + logger.log(Level.WARNING, String.format("Error parsing %s (object ID = %d), presumed corrupt", file.getName(), file.getId()), ex); //NON-NLS } catch (NoCurrentCaseException ex) { - logger.log(Level.INFO, "Exception while getting open case.", ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error processing %s (object ID = %d)", file.getName(), file.getId()), ex); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java index 470c0f8cf4..788ca39575 100644 --- a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2020 Basis Technology Corp. + * Copyright 2018-2021 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -154,7 +154,7 @@ final class TikaTextExtractor implements TextExtractor { private static final String TESSERACT_OUTPUT_FILE_NAME = "tess_output"; //NON-NLS // documents where OCR is performed - private static final ImmutableSet OCR_DOCUMENTS = ImmutableSet.of( + private static final ImmutableSet OCR_DOCUMENTS = ImmutableSet.of( "application/pdf", "application/msword", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index b796a16d26..5b2b92e49b 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -5,15 +5,10 @@ ChromeCacheExtract_adding_artifacts_msg=Chrome Cache: Adding %d artifacts for an ChromeCacheExtract_adding_extracted_files_msg=Chrome Cache: Adding %d extracted files for analysis. ChromeCacheExtract_loading_files_msg=Chrome Cache: Loading files from %s. ChromeCacheExtractor.moduleName=ChromeCacheExtractor -# {0} - module name -# {1} - row number -# {2} - table length -# {3} - cache path ChromeCacheExtractor.progressMsg={0}: Extracting cache entry {1} of {2} entries from {3} DataSourceUsage_AndroidMedia=Android Media Card DataSourceUsage_DJU_Drone_DAT=DJI Internal SD Card DataSourceUsage_FlashDrive=Flash Drive -# {0} - OS name DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0}) DataSourceUsageAnalyzer.parentModuleName=Recent Activity DefaultPriorityDomainCategorizer_searchEngineCategory=Search Engine @@ -28,7 +23,6 @@ ExtractEdge_process_errMsg_errGettingWebCacheFiles=Error trying to retrieving Ed ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer ExtractEdge_process_errMsg_webcacheFail=Failure processing Microsoft Edge WebCacheV01.dat file -# {0} - sub module name ExtractIE_executePasco_errMsg_errorRunningPasco={0}: Error analyzing Internet Explorer web history ExtractOs.androidOs.label=Android ExtractOs.androidVolume.label=OS Drive (Android) @@ -61,7 +55,6 @@ ExtractOs.windowsVolume.label=OS Drive (Windows) ExtractOs.yellowDogLinuxOs.label=Linux (Yellow Dog) ExtractOs.yellowDogLinuxVolume.label=OS Drive (Linux Yellow Dog) ExtractOS_progressMessage=Checking for OS -# {0} - sub module name ExtractPrefetch_errMsg_prefetchParsingFailed={0}: Error analyzing prefetch files ExtractPrefetch_module_name=Windows Prefetch Extractor ExtractRecycleBin_module_name=Recycle Bin @@ -88,6 +81,8 @@ ExtractZone_process_errMsg_find=A failure occured while searching for :Zone.Inde ExtractZone_progress_Msg=Extracting :Zone.Identifer files ExtractZone_Restricted=Restricted Sites Zone ExtractZone_Trusted=Trusted Sites Zone +Jumplist_adding_extracted_files_msg=Chrome Cache: Adding %d extracted files for analysis. +Jumplist_module_name=Windows Jumplist Extractor OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\n The module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web activity (sites visited, stored cookies, book marked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images.\nThe plugin is also fully functional when deployed on Windows version of Autopsy. OpenIDE-Module-Name=RecentActivity @@ -157,19 +152,13 @@ Firefox.getDlV24.errMsg.errAnalyzeFile={0}: Error while trying to analyze file:{ Firefox.getDlV24.errMsg.errParsingArtifacts={0}: Error parsing {1} Firefox web download artifacts. Progress_Message_Analyze_Registry=Analyzing Registry Files Progress_Message_Analyze_Usage=Data Sources Usage Analysis -# {0} - browserName Progress_Message_Chrome_AutoFill=Chrome Auto Fill Browser {0} -# {0} - browserName Progress_Message_Chrome_Bookmarks=Chrome Bookmarks Browser {0} Progress_Message_Chrome_Cache=Chrome Cache -# {0} - browserName Progress_Message_Chrome_Cookies=Chrome Cookies Browser {0} -# {0} - browserName Progress_Message_Chrome_Downloads=Chrome Downloads Browser {0} Progress_Message_Chrome_FormHistory=Chrome Form History -# {0} - browserName Progress_Message_Chrome_History=Chrome History Browser {0} -# {0} - browserName Progress_Message_Chrome_Logins=Chrome Logins Browser {0} Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks Progress_Message_Edge_Cookies=Microsoft Edge Cookies @@ -224,7 +213,6 @@ Recently_Used_Artifacts_Winrar=Recently opened according to WinRAR MRU Registry_System_Bam=Recently Executed according to Background Activity Moderator (BAM) RegRipperFullNotFound=Full version RegRipper executable not found. RegRipperNotFound=Autopsy RegRipper executable not found. -# {0} - file name SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}. SearchEngineURLQueryAnalyzer.moduleName.text=Search Engine SearchEngineURLQueryAnalyzer.engineName.none=NONE diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractJumpLists.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractJumpLists.java new file mode 100644 index 0000000000..fc0ef8995e --- /dev/null +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractJumpLists.java @@ -0,0 +1,237 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2021 Basis Technology Corp. + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.recentactivity; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import org.apache.poi.poifs.filesystem.DirectoryEntry; +import org.apache.poi.poifs.filesystem.DocumentEntry; +import org.apache.poi.poifs.filesystem.DocumentInputStream; +import org.apache.poi.poifs.filesystem.Entry; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.services.FileManager; +import org.sleuthkit.autopsy.coreutils.JLNK; +import org.sleuthkit.autopsy.coreutils.JLnkParser; +import org.sleuthkit.autopsy.coreutils.JLnkParserException; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DerivedFile; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; + +/** + * Extract the LNK files from the jumplists and save them to ModuleOutput\RecentActivity\Jumplists + * and then add them back into the case as a dervived file. + */ +final class ExtractJumpLists extends Extract { + + private static final Logger logger = Logger.getLogger(ExtractJumpLists.class.getName()); + + private IngestJobContext context; + + private static final String JUMPLIST_TSK_COMMENT = "Jumplist File"; + private static final String RA_DIR_NAME = "RecentActivity"; //NON-NLS + private static final String MODULE_OUTPUT_DIR = "ModuleOutput"; //NON-NLS + private static final String AUTOMATIC_DESTINATIONS_FILE_DIRECTORY = "%/AppData/Roaming/Microsoft/Windows/Recent/AutomaticDestinations/"; + private static final String JUMPLIST_DIR_NAME = "jumplists"; //NON-NLS + private static final String VERSION_NUMBER = "1.0.0"; //NON-NLS + private String moduleName; + private FileManager fileManager; + private final IngestServices services = IngestServices.getInstance(); + + @Messages({ + "Jumplist_module_name=Windows Jumplist Extractor", + "Jumplist_adding_extracted_files_msg=Chrome Cache: Adding %d extracted files for analysis." + }) + ExtractJumpLists() { + super(Bundle.Jumplist_module_name()); + } + + @Override + void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { + + this.context = context; + moduleName = Bundle.Jumplist_module_name(); + fileManager = currentCase.getServices().getFileManager(); + long ingestJobId = context.getJobId(); + + List jumpListFiles = extractJumplistFiles(dataSource, ingestJobId); + + if (jumpListFiles.isEmpty()) { + return; + } + + if (context.dataSourceIngestIsCancelled()) { + return; + } + + List derivedFiles = new ArrayList<>(); + String derivedPath = null; + String baseRaTempPath = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), JUMPLIST_DIR_NAME + "_" + dataSource.getId(), ingestJobId); + for (AbstractFile jumplistFile : jumpListFiles) { + if (!jumplistFile.getName().toLowerCase().contains("-slack") && !jumplistFile.getName().equals("..") && + !jumplistFile.getName().equals(".") && jumplistFile.getSize() > 0) { + String jlFile = Paths.get(baseRaTempPath, jumplistFile.getName() + "_" + jumplistFile.getId()).toString(); + String moduleOutPath = Case.getCurrentCase().getModuleDirectory() + File.separator + RA_DIR_NAME + File.separator + JUMPLIST_DIR_NAME + "_" + dataSource.getId() + File.separator + jumplistFile.getName() + "_" + jumplistFile.getId(); + derivedPath = RA_DIR_NAME + File.separator + JUMPLIST_DIR_NAME + "_" + dataSource.getId() + File.separator + jumplistFile.getName() + "_" + jumplistFile.getId(); + File jlDir = new File(moduleOutPath); + if (jlDir.exists() == false) { + boolean dirMade = jlDir.mkdirs(); + if (!dirMade) { + logger.log(Level.WARNING, "Error creating directory to store Jumplist LNK files %s", moduleOutPath); //NON-NLS + continue; + } + } + derivedFiles.addAll(extractLnkFiles(jlFile, moduleOutPath, jumplistFile, derivedPath)); + } + } + + // notify listeners of new files and schedule for analysis + progressBar.progress(String.format(Bundle.Jumplist_adding_extracted_files_msg(), derivedFiles.size())); + derivedFiles.forEach((derived) -> { services.fireModuleContentEvent(new ModuleContentEvent(derived)); }); + context.addFilesToJob(derivedFiles); + + } + + /** + * Find jumplist and extract jumplist files to temp directory + * + * @return - list of jumplist abstractfiles or empty list + */ + private List extractJumplistFiles(Content dataSource, Long ingestJobId) { + List jumpListFiles = new ArrayList<>();; + List tempJumpListFiles = new ArrayList<>();; + + FileManager fileManager = Case.getCurrentCase().getServices().getFileManager(); + + try { + tempJumpListFiles = fileManager.findFiles(dataSource, "%", AUTOMATIC_DESTINATIONS_FILE_DIRECTORY); //NON-NLS + if (!tempJumpListFiles.isEmpty()) { + jumpListFiles.addAll(tempJumpListFiles); + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Unable to find jumplist files.", ex); //NON-NLS + return jumpListFiles; // No need to continue + } + + for (AbstractFile jumpListFile : jumpListFiles) { + + if (context.dataSourceIngestIsCancelled()) { + return jumpListFiles; + } + + if (!jumpListFile.getName().toLowerCase().contains("-slack") && !jumpListFile.getName().equals("..") && + !jumpListFile.getName().equals(".") && jumpListFile.getSize() > 0) { + String fileName = jumpListFile.getName() + "_" + jumpListFile.getId(); + String baseRaTempPath = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), JUMPLIST_DIR_NAME+ "_" + dataSource.getId(), ingestJobId); + String jlFile = Paths.get(baseRaTempPath, fileName).toString(); + try { + ContentUtils.writeToFile(jumpListFile, new File(jlFile)); + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Unable to write %s to temp directory. File name: %s", fileName, jlFile), ex); //NON-NLS + } + } + } + + return jumpListFiles; + + } + + /* + * Read each jumplist file and extract the lnk files to moduleoutput + */ + private List extractLnkFiles(String jumpListFile, String moduleOutPath, AbstractFile jumpListAbsFile, String derivedPath) { + + List derivedFiles = new ArrayList<>(); + DerivedFile derivedFile; + String lnkFileName = ""; + + try (POIFSFileSystem fs = new POIFSFileSystem(new File(jumpListFile))) { + DirectoryEntry root = fs.getRoot(); + for (Entry entry : root) { + if (entry instanceof DirectoryEntry) { + //If this data structure needed to recurse this is where it would do it but jumplists do not need to at this time + continue; + } else if (entry instanceof DocumentEntry) { + String jmpListFileName = entry.getName(); + int fileSize = ((DocumentEntry) entry).getSize(); + + if (fileSize > 0) { + try (DocumentInputStream stream = fs.createDocumentInputStream(jmpListFileName)) { + byte[] buffer = new byte[stream.available()]; + stream.read(buffer); + + JLnkParser lnkParser = new JLnkParser(fs.createDocumentInputStream(jmpListFileName), fileSize); + JLNK lnk = lnkParser.parse(); + lnkFileName = lnk.getBestName() + ".lnk"; + File targetFile = new File(moduleOutPath + File.separator + entry.getName() + "-" + lnkFileName); + String derivedFileName = MODULE_OUTPUT_DIR + File.separator + derivedPath + File.separator + entry.getName() + "-" + lnkFileName; + OutputStream outStream = new FileOutputStream(targetFile); + outStream.write(buffer); + outStream.close(); + derivedFile = fileManager.addDerivedFile(lnkFileName, derivedFileName, + fileSize, + 0, + 0, + 0, + 0, // TBD + true, + jumpListAbsFile, + "", + moduleName, + VERSION_NUMBER, + "", + TskData.EncodingType.NONE); + derivedFiles.add(derivedFile); + + } catch (IOException | JLnkParserException e) { + logger.log(Level.WARNING, String.format("No such document, or the Entry represented by documentName is not a DocumentEntry link file is %s", jumpListFile), e); //NON-NLS + } + } + } else { + // currently, either an Entry is a DirectoryEntry or a DocumentEntry, + // but in the future, there may be other entry subinterfaces. + // The internal data structure certainly allows for a lot more entry types. + continue; + } + } + } catch (IOException | TskCoreException ex) { + logger.log(Level.WARNING, String.format("Error lnk parsing the file to get recent files $s", jumpListFile), ex); //NON-NLS + } + + return derivedFiles; + + } + +} diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractPrefetch.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractPrefetch.java index 3b0239fcc5..c63adccd5a 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractPrefetch.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractPrefetch.java @@ -88,6 +88,16 @@ final class ExtractPrefetch extends Extract { super(Bundle.ExtractPrefetch_module_name()); } + /** + * Get the temp folder name. + * + * @param dataSource Current data source + * @return The folder name + */ + private String getPrefetchTempFolder(Content dataSource) { + return dataSource.getId() + "-" + PREFETCH_PARSER_DB_FILE; + } + @Override void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { @@ -116,9 +126,9 @@ final class ExtractPrefetch extends Extract { return; } - String modOutFile = modOutPath + File.separator + dataSource.getName() + "-" + PREFETCH_PARSER_DB_FILE; + String modOutFile = modOutPath + File.separator + getPrefetchTempFolder(dataSource); try { - String tempDirPath = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), dataSource.getName() + "-" + PREFETCH_DIR_NAME, ingestJobId); + String tempDirPath = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), getPrefetchTempFolder(dataSource), ingestJobId); parsePrefetchFiles(prefetchDumper, tempDirPath, modOutFile, modOutPath); File prefetchDatabase = new File(modOutFile); if (prefetchDatabase.exists()) { @@ -159,7 +169,7 @@ final class ExtractPrefetch extends Extract { String ext = FilenameUtils.getExtension(origFileName); String baseName = FilenameUtils.getBaseName(origFileName); String fileName = escapeFileName(String.format("%s_%d.%s", baseName, pFile.getId(), ext)); - String baseRaTempPath = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), dataSource.getName() + "-" + PREFETCH_DIR_NAME, ingestJobId); + String baseRaTempPath = RAImageIngestModule.getRATempPath(Case.getCurrentCase(), getPrefetchTempFolder(dataSource), ingestJobId); String prefetchFile = Paths.get(baseRaTempPath, fileName).toString(); try { ContentUtils.writeToFile(pFile, new File(prefetchFile)); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index e3df208180..f51bcbd0f4 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -91,6 +91,7 @@ import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAM import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH; import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HOME_DIR; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataArtifact; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.HostManager; @@ -1769,6 +1770,7 @@ class ExtractRegistry extends Extract { */ void createShellBagArtifacts(AbstractFile regFile, List shellbags) throws TskCoreException { List artifacts = new ArrayList<>(); + List dataArtifacts = new ArrayList<>(); try { for (ShellBag bag : shellbags) { Collection attributes = new ArrayList<>(); @@ -1796,11 +1798,14 @@ class ExtractRegistry extends Extract { attributes.add(new BlackboardAttribute(TSK_DATETIME_ACCESSED, getName(), time)); } - artifacts.add(createArtifactWithAttributes(getShellBagArtifact(), regFile, attributes)); + BlackboardArtifact artifact = createArtifactWithAttributes(getShellBagArtifact(), regFile, attributes); + artifacts.add(artifact); + dataArtifacts.add((DataArtifact)artifact); } } finally { if(!context.dataSourceIngestIsCancelled()) { - postArtifacts(artifacts); + postArtifacts(artifacts); + context.addDataArtifactsToJob(dataArtifacts); } } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java index 290f406a8a..bf64688408 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RAImageIngestModule.java @@ -84,8 +84,10 @@ public final class RAImageIngestModule implements DataSourceIngestModule { Extract prefetch = new ExtractPrefetch(); Extract webAccountType = new ExtractWebAccountType(); Extract messageDomainType = new DomainCategoryRunner(); + Extract jumpList = new ExtractJumpLists(); extractors.add(recycleBin); + extractors.add(jumpList); extractors.add(recentDocuments); extractors.add(registry); // needs to run before the DataSourceUsageAnalyzer extractors.add(osExtract); // this needs to run before the DataSourceUsageAnalyzer diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java index d9ad0fbfb8..e8e7240fee 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/RecentDocumentsByLnk.java @@ -22,6 +22,7 @@ */ package org.sleuthkit.autopsy.recentactivity; +import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; @@ -29,6 +30,7 @@ import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import java.util.Collection; +import java.util.HashMap; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.JLNK; import org.sleuthkit.autopsy.coreutils.JLnkParser; @@ -87,6 +89,7 @@ class RecentDocumentsByLnk extends Extract { dataFound = true; List bbartifacts = new ArrayList<>(); + HashMap recentFileMap = new HashMap<>(); for (AbstractFile recentFile : recentFiles) { if (context.dataSourceIngestIsCancelled()) { break; @@ -111,6 +114,8 @@ class RecentDocumentsByLnk extends Extract { Collection bbattributes = new ArrayList<>(); String path = lnk.getBestPath(); + if (recentFileMap.get(path + File.separator + recentFile.getName()) == null) { + recentFileMap.put(path + File.separator + recentFile.getName(), recentFile.getName()); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH, NbBundle.getMessage(this.getClass(), "RecentDocumentsByLnk.parentModuleName.noSpace"), @@ -136,6 +141,7 @@ class RecentDocumentsByLnk extends Extract { logger.log(Level.SEVERE, String.format("Failed to create TSK_RECENT_OBJECT artifact for file %d", recentFile.getId()), ex); } } + } if (!context.dataSourceIngestIsCancelled()) { postArtifacts(bbartifacts); diff --git a/build-windows-installer.xml b/build-windows-installer.xml index 5bf9bdb67a..732d72bad9 100644 --- a/build-windows-installer.xml +++ b/build-windows-installer.xml @@ -173,7 +173,7 @@ - + diff --git a/docs/doxygen-user_fr/adHocKeywordSearch.dox b/docs/doxygen-user_fr/adHocKeywordSearch.dox index 315f903225..e6eab014f2 100644 --- a/docs/doxygen-user_fr/adHocKeywordSearch.dox +++ b/docs/doxygen-user_fr/adHocKeywordSearch.dox @@ -80,7 +80,7 @@ Les résultats apparaitront dans une visionneuse de résultats distincte pour ch \section ad_hoc_kw_lists Keyword Lists (Listes de mots-clés) -En plus d'être sélectionnées lors de l'acquisition, les listes de mots-clés peuvent également être exécutées via le bouton "Keyword Lists". Pour plus d'informations sur la configuration de ces listes de mots clés, consultez la section \ref keywordListsTab de la documentation du module d'acquisition. +En plus d'être sélectionnées lors de l'acquisition, les listes de mots-clés peuvent également être exécutées via le bouton "Keyword Lists". Pour plus d'informations sur la configuration de ces listes de mots clés, consultez la section \ref keyword_keywordListsTab de la documentation du module d'acquisition. Les listes créées à l'aide de la boîte de dialogue "Keyword Search Configuration" peuvent être recherchées manuellement par l'utilisateur en appuyant sur le bouton "Keyword Lists" et en cochant les cases correspondant aux listes à rechercher. La recherche peut être limitée à certaines sources de données uniquement en cochant la case située en bas, puis en mettant en surbrillance les sources de données dans lesquelles effectuer la recherche. Plusieurs sources de données peuvent être sélectionnées en utilisant les touches Maj+clic gauche ou Ctrl+clic gauche. Une fois que tout a été configuré, appuyez sur "Search" pour lancer la recherche. La case à cocher "Save search results" détermine si les résultats de la recherche seront enregistrés dans la base de données du cas. diff --git a/docs/doxygen-user_fr/central_repo.dox b/docs/doxygen-user_fr/central_repo.dox index 3c4d1476b1..d6f985ef4b 100644 --- a/docs/doxygen-user_fr/central_repo.dox +++ b/docs/doxygen-user_fr/central_repo.dox @@ -1,4 +1,4 @@ -/*! \page central_repo_page Référentiel central +/*! \page central_repo_page Référentiel central [TOC] @@ -111,6 +111,10 @@ Descriptions des types de propriétés: - Les propriétés d'ICCID ne sont actuellement créées que par des modules Autopsy personnalisés. - Credit Card - Les propriétés de carte de crédit sont créées par le module \ref keyword_search_page. +- OS Account + - Les propriétés de comptes de système d'exploitation sont créés par le processeur de source de données d'image disque et le module \ref recent_activity_page. +- Installed Programs + - Les propriétés des programmes installés sont créées principalement par le module \ref recent_activity_page. - App-specific Accounts (Facebook, Twitter, etc...) - Ces propriétés proviennent principalement du module \ref android_analyzer_page. @@ -146,7 +150,7 @@ Il existe trois paramètres pour le module d'acquisition "Central Repository":

  • Save items to the Central Repository - Cette option ne doit être désélectionnée que dans les rares cas où vous ne souhaitez pas ajouter de propriétés de la source de données actuelle au référentiel central, mais souhaitez quand même signaler les occurrences passées.
  • Flag items previously tagged as notable - L'activation de cette option entraîne la création d'artefacts d'éléments/fichiers intéressants lorsque des propriétés correspondant à celles précédemment marquées sont trouvées. Voir la section suivante \ref cr_tagging pour plus de détails. -
  • Flag previously seen devices - Lorsque cette option est activée, un artefact "Interesting Item" sera créé si une propriété liée au périphérique (USB, adresse MAC, IMSI, IMEI, ICCID) est détectée et se trouve déjà dans le référentiel central, qu'elle ait été ou non signalée. +
  • Flag previously seen devices and users - Lorsque cette option est activée, un artefact "Interesting Item" sera créé si une propriété liée au périphérique (USB, adresse MAC, IMSI, IMEI, ICCID) ou un compte de système d'exploitation sont détectés et se trouvent déjà dans le référentiel central, qu'ils aient été ou non marqués.
    • \subsection cr_tagging Marquage de fichiers et d'artefacts @@ -228,4 +232,4 @@ Dans le cas du type de numéro de téléphone corrélable, l'arborescence "Inter -*/ +*/ \ No newline at end of file diff --git a/docs/doxygen-user_fr/content_viewer.dox b/docs/doxygen-user_fr/content_viewer.dox index edd3e46f25..702420df08 100644 --- a/docs/doxygen-user_fr/content_viewer.dox +++ b/docs/doxygen-user_fr/content_viewer.dox @@ -75,9 +75,9 @@ Les fichiers des ruches de la Base de registre peuvent être affichés dans un f \image html content_viewer_registry.png -\section cv_metadata File Metadata +\section cv_metadata File Metadata / Source File Metadata -L'onglet "File Metadata" affiche des informations de base sur le fichier, telles que le type, la taille et le hachage. Il affiche également la sortie de l'outil istat du Sleuth Kit. +L'onglet "File Metadata" affiche des informations de base sur le fichier sélectionné ou le fichier associé au résultat, telles que le type, la taille et le hachage. Il affiche également la sortie de l'outil istat du Sleuth Kit. \image html content_viewer_metadata.png @@ -87,14 +87,20 @@ L'onglet "OS Accounts" affiche des informations sur le compte du système d'expl \image html content_viewer_os_account.png -\section cv_results Results +\section cv_results Data Artifacts -L'onglet "Results" est activé lors de la sélection d'éléments avec des résultats associés tels que des hits sur des mots clés, des journaux d'appels et des messages. Les champs exacts affichés dépendent du type de résultat. Les deux images ci-dessous montrent l'onglet "Results" pour un journal d'appels et un signet Web. +L'onglet "Data Artifacts" affiche les artefacts associés à l'élément sélectionné dans la visionneuse de résultats, tels que des signets Web, des journaux d'appels et des messages. Les champs exacts affichés dépendent du type d'artefact de données. Les deux images ci-dessous montrent l'onglet "Data Artifacts" pour un journal d'appels et un signet Web. \image html content_viewer_results_call.png
    \image html content_viewer_results_bookmark.png +\section cv_analysis_results Analysis Results + +L'onglet "Analysis Results" affiche tous les résultats d'analyse associés à l'élément sélectionné dans la visionneuse de résultats. Si vous sélectionnez un résultat d'analyse, la liste défilera automatiquement jusqu'à ce résultat. Ces résultats d'analyse proviennent de données telles que les résultats de recherches de hachage, les éléments intéressants et les résultats de recherches de mots clés. L'image ci-dessous montre les résultats de l'analyse des catégories Web. + +\image html content_viewer_analysis_result_webcat.png + \section cv_context Context L'onglet "Context" affiche des informations sur l'origine d'un fichier et vous permet d'accéder au résultat d'origine. Par exemple, il peut afficher l'URL des fichiers téléchargés et le message électronique auquel un fichier était joint. Dans l'image ci-dessous, vous pouvez voir le contexte d'une image qui a été envoyée en tant que pièce jointe à un e-mail. @@ -114,4 +120,4 @@ L'onglet "Other Occurrences" affiche d'autres instances de ce fichier ou résult \image html content_viewer_other_occurrences.png -*/ +*/ \ No newline at end of file diff --git a/docs/doxygen-user_fr/file_search.dox b/docs/doxygen-user_fr/file_search.dox index fa6b210f1f..6735f06885 100644 --- a/docs/doxygen-user_fr/file_search.dox +++ b/docs/doxygen-user_fr/file_search.dox @@ -1,4 +1,4 @@ -/*! \page file_search_page Recherche de fichier +/*! \page file_search_page Recherche de fichier [TOC] @@ -21,7 +21,7 @@ ou sélectionnez le menu "Tools", "File Search by Attributes". Il existe plusieurs catégories que vous pouvez utiliser pour filtrer et afficher les répertoires et les fichiers dans les images du cas ouvert actuellement. Ces catégories sont: \li Name: -Recherchez tous les fichiers et répertoires dont le nom contient le modèle donné. +Recherchez tous les fichiers et répertoires dont le nom contient le modèle donné. La recherche porte uniquement sur le nom du fichier/répertoire et ne prends pas en compte le chemin parent. Remarque: il ne prend pas en charge la correspondance d'expressions régulières et de mots clés. \li Size: Recherchez tous les fichiers et répertoires dont la taille correspond au modèle donné. Le motif peut être "equal to" (égal à), "greater than" (supérieur à) et "less than" (inférieur à). L'unité pour la taille peut être "Octet(s)", "Ko", "Mo", "Go" et "To". @@ -32,7 +32,7 @@ Recherchez tous les fichiers avec le hachage MD5 donné. \li Date: Recherchez tous les fichiers et répertoires dont la propriété "date" est comprise dans la plage de dates indiquée. Les propriétés de date sont "Modified Date" (Date de modification), "Accessed Date" (Date d'accès), "Changed Date" (Date de changement), and "Created Date" (Date de création). Vous devez également spécifier le fuseau horaire de la date donnée. \li Known Status: -Recherchez tous les fichiers et répertoires dont l'état est reconnu comme "Unknown" (Inconnu), "Known" (Connu) ou "Notable" (Connu défavorablement). Pour plus d'informations sur ces états, consultez la page \ref hash_db_page. +Recherchez tous les fichiers dont l'état est reconnu comme "Unknown" (Inconnu), "Known" (Connu) ou "Notable" (Connu défavorablement). Pour plus d'informations sur ces états, consultez la page \ref hash_db_page. Pour utiliser l'un de ces filtres, cochez la case à côté de la catégorie et cliquez sur le bouton "Search" pour démarrer le processus de recherche. Le résultat apparaîtra dans la visionneuse de résultats. \li Data Source: Rechercher uniquement dans la source de données spécifiée au lieu de l'ensemble du cas. Notez que plusieurs sources de données peuvent être sélectionnées en maintenant MAJ ou CTRL pendant la sélection. diff --git a/docs/doxygen-user_fr/hosts.dox b/docs/doxygen-user_fr/hosts.dox index d1a0b30a08..eb13f9df52 100644 --- a/docs/doxygen-user_fr/hosts.dox +++ b/docs/doxygen-user_fr/hosts.dox @@ -1,4 +1,4 @@ -/*! \page host_page Hosts (Hôtes) +/*! \page host_page Hosts (Hôtes) [TOC] @@ -19,7 +19,7 @@ Les hôtes sont affichés dans l'\ref tree_viewer_page. En fonction des \ref vie \subsection host_os_accounts Comptes de système d'exploitation -Les comptes de système d'exploitation peuvent être affichés dans le nœud "OS Accounts" sous "Results". Chaque compte de système d'exploitation est associé à un hôte et les informations sur l'hôte sont affichées dans l'onglet "OS Accounts" de la visionneuse de contenu. +Les comptes de système d'exploitation peuvent être affichés dans le nœud "OS Accounts" de l'arborescence. Chaque compte de système d'exploitation est associé à un hôte et les informations sur l'hôte sont affichées dans l'onglet "OS Accounts" de la visionneuse de contenu. \image html host_os_accounts.png diff --git a/docs/doxygen-user_fr/images/central_repo_ingest_settings.png b/docs/doxygen-user_fr/images/central_repo_ingest_settings.png index dec839f689..79feba954e 100644 Binary files a/docs/doxygen-user_fr/images/central_repo_ingest_settings.png and b/docs/doxygen-user_fr/images/central_repo_ingest_settings.png differ diff --git a/docs/doxygen-user_fr/images/content_viewer_results_bookmark.png b/docs/doxygen-user_fr/images/content_viewer_results_bookmark.png index 2c0d3cb736..cbefb1df56 100644 Binary files a/docs/doxygen-user_fr/images/content_viewer_results_bookmark.png and b/docs/doxygen-user_fr/images/content_viewer_results_bookmark.png differ diff --git a/docs/doxygen-user_fr/images/content_viewer_results_call.png b/docs/doxygen-user_fr/images/content_viewer_results_call.png index 3bc104af0a..843a68cd24 100644 Binary files a/docs/doxygen-user_fr/images/content_viewer_results_call.png and b/docs/doxygen-user_fr/images/content_viewer_results_call.png differ diff --git a/docs/doxygen-user_fr/images/keyword-search-configuration-dialog-general.PNG b/docs/doxygen-user_fr/images/keyword-search-configuration-dialog-general.PNG index 4dbb566faa..dadd9f4e71 100644 Binary files a/docs/doxygen-user_fr/images/keyword-search-configuration-dialog-general.PNG and b/docs/doxygen-user_fr/images/keyword-search-configuration-dialog-general.PNG differ diff --git a/docs/doxygen-user_fr/images/keyword-search-ingest-settings.PNG b/docs/doxygen-user_fr/images/keyword-search-ingest-settings.PNG index d7c0b8e2fc..85c6378d60 100644 Binary files a/docs/doxygen-user_fr/images/keyword-search-ingest-settings.PNG and b/docs/doxygen-user_fr/images/keyword-search-ingest-settings.PNG differ diff --git a/docs/doxygen-user_fr/keyword_search.dox b/docs/doxygen-user_fr/keyword_search.dox index 5fd79997cd..cdc0061453 100644 --- a/docs/doxygen-user_fr/keyword_search.dox +++ b/docs/doxygen-user_fr/keyword_search.dox @@ -1,4 +1,4 @@ -/*! \page keyword_search_page Keyword Search (Recherches par mots clés) +/*! \page keyword_search_page Keyword Search (Recherches par mots clés) [TOC] @@ -18,11 +18,11 @@ Référez vous à la page \ref ad_hoc_keyword_search_page pour plus de détails \section keyword_search_configuration_dialog Configuration de la recherche par mot-clé L'option de configuration de la recherche par mot-clé ("Keyword Search") comporte trois onglets, chacun ayant son propre objectif: -\li L'\ref keywordLists est utilisé pour ajouter, supprimer et modifier des listes de recherche par mot-clé. -\li L'\ref stringExtraction est utilisé pour activer les scripts de langage et le type d'extraction. -\li L'\ref generalSettings est utilisé pour configurer les horaires d'acquisition et afficher les informations. +\li L'\ref keyword_keywordListsTab est utilisé pour ajouter, supprimer et modifier des listes de recherche par mot-clé. +\li L'\ref keyword_stringExtractionTab est utilisé pour activer les scripts de langage et le type d'extraction. +\li L'\ref keyword_generalSettingsTab est utilisé pour configurer les horaires d'acquisition et afficher les informations. -\subsection keywordLists Onglet "Lists" +\subsection keyword_keywordListsTab Onglet "Lists" L'onglet "Lists" est utilisé pour créer/importer et ajouter du contenu aux listes de mots clés. Pour créer une liste, sélectionnez le bouton "New List" et choisissez un nom pour la nouvelle liste de mots clés. Une fois la liste créée, des mots clés peuvent y être ajoutés (voir la section \ref ad_hoc_kw_types_section pour plus d'informations sur les types de mots-clés). Des listes peuvent être ajoutées au processus d'acquisition de la recherche par mot-clé; les recherches auront lieu à intervalles réguliers au fur et à mesure que le contenu est ajouté à l'index. @@ -40,7 +40,7 @@ Sous la liste "Keywords", vous pouvez solliciter la réception d'un messages dan \image html keyword-search-inbox.PNG -\subsection stringExtraction Onglet "String Extraction" +\subsection keyword_stringExtractionTab Onglet "String Extraction" Le paramètre "String Extraction" définit comment les chaînes de caractères sont extraites des fichiers dont le texte ne peut pas être extrait normalement car les formats de ces fichier ne sont pas pris en charge. C'est le cas des fichiers binaires arbitraires (tels que les fichiers d'échanges) et des morceaux d'espace non alloué qui représentent des fichiers supprimés. Lorsque nous extrayons des chaînes de caractères de fichiers binaires, nous devons interpréter les séquences d'octets comme du texte différemment en fonction du codage de texte possible et du script/langage utilisé. Dans de nombreux cas, nous ne savons pas à l'avance dans quel encodage/langue spécifique le texte est encodé. Cependant, cela peut être intéressant si l'enquêteur recherche une langue spécifique, car en sélectionnant moins de langues, les performances d'indexation seront améliorées et le nombre des faux positifs seront réduits. @@ -50,20 +50,36 @@ Le paramètre par défaut consiste à rechercher uniquement les chaînes anglais L'utilisateur peut également utiliser en premier le "String Viewer" et essayer différents paramètres de script/langue, et voir quels paramètres donnent des résultats satisfaisants pour le type de texte pertinent pour l'enquête. Ensuite, ce même paramètre qui fonctionne pour l'enquête peut être appliqué au module d'acquisition de recherche par mot-clé. -\subsection generalSettings Onglet "General" +\subsection keyword_generalSettingsTab Onglet "General" \image html keyword-search-configuration-dialog-general.PNG -### Prise en charge du NIST NSRL +\subsubsection keyword_nsrl Prise en charge du NIST NSRL Le module d'acquisition "Hash Lookup" peut être configuré pour utiliser l'ensemble de hachage NIST NSRL de fichiers connus. L'onglet "General" de la boîte de dialogue de configuration avancée de la recherche par mot-clé contient une option permettant d'ignorer l'indexation par mot-clé et de rechercher des fichiers précédemment marqués comme "connus" ("Known") et sans intérêt. La sélection de cette option peut réduire considérablement la taille de l'index et améliorer les performances d'acquisition. Dans la plupart des cas, l'utilisateur n'a pas besoin de rechercher par mot-clé les fichiers "connus". -### Fréquence de mise à jour des résultats lors de l'acquisition +\subsubsection keyword_update_freq Fréquence de mise à jour des résultats lors de l'acquisition Pour contrôler la fréquence à laquelle les recherches sont exécutées pendant l'acquisition, l'utilisateur peut ajuster le paramètre de synchronisation disponible dans l'onglet "General" de la boîte de dialogue de configuration avancée de la recherche par mot-clé. La réduction du nombre de minutes entraînera des mises à jour d'index et des recherches plus fréquentes et l'utilisateur pourra voir les résultats davantage en temps réel. Cependant, des mises à jour plus fréquentes peuvent affecter les performances globales, en particulier sur les systèmes peu performants, et peuvent potentiellement allonger le temps total nécessaire à l'acquisition. On peut également choisir de ne pas effectuer de recherches périodiques. Cela accélérera l'acquisition. Les utilisateurs qui choisissent cette option peuvent exécuter leurs recherches par mots-clés une fois que l'index de recherche par mots-clés est complet. -### Reconnaissance optique de caractères (OCR) -Il existe également un paramètre pour activer le Optical Character Recognition (OCR). Si cette option est activée, le texte peut être extrait des types d'images pris en charge. L'activation de cette fonctionnalité rendra le module de recherche par mot-clé plus long à exécuter et les résultats ne sont pas parfaits. La deuxième case à cocher peut accélérer l'exécution de l'OCR en ne traitant que les grandes images et les images extraites de documents. +\section keyword_usage Utilisation du module + +Les recherche peuvent être exécutées manuellement par l'utilisateur à tout moment, tant qu'il existe des fichiers déjà indexés et prêts à être analysés. Une recherche effectuée avant la fin de l'indexation ne s'exécutera naturellement que les index déjà compilés. + +Voir la page sur les \ref ingest_page "Ingest Modules" pour plus d'information sur les modules d'acquisition en général. + +Une fois qu'il y a des fichiers dans l'index, la \ref ad_hoc_keyword_search_page sera disponible pour une recherche manuelle à tout moment. + +\subsection keyword_ingest_settings Paramétrages + +Les options de paramétrages du module de recherches par mots clés permettent à l'utilisateur d'activer ou de désactiver les expressions de recherche intégrées spécifiques : Phone Numbers (numéros de téléphone), IP Addresses (adresses IP), Email Addresses (adresses mail), et URLs. En utilisant le bouton "Global Settings" (expliqué ci-dessous), on peut ajouter des ensembles de mots clés personnalisés. + +\image html keyword-search-ingest-settings.PNG + +\subsubsection keyword_ocr Optical Character Recognition (reconnaissance optique de caractères) +\anchor keyword_search_ocr_config + +Il y a aussi un paramètre pour activer la reconnaissance optique de caractères (ou Optical Character Recognition - OCR). S'il est activé, le texte peut être extrait des types d'images pris en charge. L'activation de cette fonctionnalité rendra l'exécution du module de recherche par mot-clé plus longue et les résultats pourront ne pas être parfaits. Voici un exemple d'image contenant du texte: @@ -73,7 +89,12 @@ L'onglet "Indexed Text" affiche les résultats lors de l'exécution du module de \image html keyword-search-ocr-indexed-text.png -\anchor keyword_search_ocr_config +Les deux options liées à l'OCR sont les suivantes: +
      +
    • Only index text extracted using OCR. Cela n'indexera que le texte détecté par l'OCR et empêchera l'indexation de texte trouvé dans les fichiers texte, les documents, etc... +
    • Only process PDFs, MS Office docs and images which are over 100KB in size or extracted from another file. Avec cette option sélectionnée, l'OCR ne sera effectuée que sur les images de plus de 100 Ko et les documents PDF/Office. Il fonctionnera également sur des images de toute taille extraites d'un autre fichier. +
    + Par défaut, l'OCR n'est configuré que pour le texte anglais. Sa configuration dépend de la présence de fichiers de langue (appelés fichiers "traineddata") qui existent dans un endroit qu'Autopsy peut atteindre. Pour ajouter la prise en charge de plusieurs langues, vous devrez télécharger des "traineddata" supplémentaires et les déplacer au bon endroit. Les étapes suivantes décrivent ce processus: @@ -88,28 +109,7 @@ et les déplacer au bon endroit. Les étapes suivantes décrivent ce processus: Les fichiers de langue seront désormais pris en charge lorsque l'OCR est activé dans les paramètres de "Keyword Search". - - -
    -Utilisation du module -====== -Les requêtes de recherche peuvent être exécutées manuellement par l'utilisateur à tout moment, à condition que certains fichiers soient déjà indexés et prêts à être recherchés. La recherche avant que l'indexation ne soit terminée ne prendra naturellement en compte que les index déjà compilés. - -Voir la page \ref ingest_page "Modules d'acquisition" pour plus d'informations sur l'acquisition en général. - -Une fois qu'il y a des fichiers dans l'index, la \ref ad_hoc_keyword_search_page sera disponible pour une recherche manuelle à tout moment. - - - -Paramètres d'acquisition ------- -Les paramètres d'acquisition du module "Keyword Search" permettent à l'utilisateur d'activer ou de désactiver les expressions de recherche intégrées spécifiques : Phone Numbers, IP Addresses, Email Addresses, and URLs. En utilisant le bouton "Global Settings" (voir ci-dessous), on peut ajouter des groupes de mots clés personnalisés. - -\image html keyword-search-ingest-settings.PNG - - -Voir les résultats ------- +\section keyword_results Voir les résultats Le module "Keyword Search" enregistrera les résultats de la recherche, que celle-ci ait été effectuée par le processus d'acquisition ou manuellement par l'utilisateur. Les résultats enregistrés sont disponibles dans l'arborescence des répertoires dans le panneau de gauche. @@ -118,4 +118,4 @@ Les résultats des mots clés apparaîtront dans l'arborescence sous "Keyword Hi \image html keyword_results.PNG -*/ +*/ \ No newline at end of file diff --git a/docs/doxygen-user_fr/multi-user/installSolr.dox b/docs/doxygen-user_fr/multi-user/installSolr.dox index ce4b8c7899..5bc4aa3f0c 100644 --- a/docs/doxygen-user_fr/multi-user/installSolr.dox +++ b/docs/doxygen-user_fr/multi-user/installSolr.dox @@ -300,7 +300,7 @@ En bref:
  • Les validations "souples" ne transfèrent pas les documents nouvellement indexés sur le disque, mais ils les rendent "visibles" pour la recherche.
-Par défaut (lors de l'utilisation d'AutopsyConfig), les serveurs Solr effectuent une validation "dure" toutes les 5 minutes et rendent également les documents nouvellement indexés "visibles" pour la recherche. Ces opérations peuvent être coûteuses en ressources lorsqu'elles sont effectuées sur un index volumineux situé sur un lecteur réseau partagé. Dans cette situation, il peut être très intéressant de modifier la configuration de Solr (située dans \c "REPERTOIRE_INSTALLATION_SOLR\server\solr\configsets\AutopsyConfig\conf\solrconfig.xml") avec les changements suivantes : +Par défaut (lors de l'utilisation d'AutopsyConfig), les serveurs Solr effectuent une validation "dure" toutes les 5 minutes et rendent également les documents nouvellement indexés "visibles" pour la recherche. Ces opérations peuvent être coûteuses en ressources lorsqu'elles sont effectuées sur un index volumineux situé sur un lecteur réseau partagé. Dans cette situation, il peut être très intéressant de modifier la configuration de Solr (située dans \c "REPERTOIRE_INSTALLATION_SOLR\server\solr\configsets\AutopsyConfig\conf\solrconfig.xml") avec les changements suivants :
  1. Modifiez les validations "dures" pour transférer systématiquement les documents nouvellement créés toutes les 5 minutes sans les rendre "visibles". Ceci peut être fixé en définissant "openSearcher" sur "false" dans la section "autoCommit" du fichier de configuration Solr.
  2. Activez les validations "souples" à effectuer toutes les 30 minutes, rendant ainsi les documents nouvellement indexés "visibles" pour une recherche toutes les 30 minutes. Ceci peut être fixé en activant la section "autoSoftCommit" du fichier de configuration Solr. L'inconvénient est que la recherche du dernier document peut prendre jusqu'à 30 minutes. Gardez à l'esprit que cela n'a d'incidence que dans le cas où un analyste recherche un dossier alors que l'acquisition est toujours en cours. Autopsy effectue automatiquement une validation une fois l'acquisition terminée afin que tous les documents soient immédiatement visibles à ce moment-là. @@ -319,4 +319,4 @@ Notez bien cependant que cela a pour effet secondaire de créer des erreurs Comm \image html solr_transaction_log_errors.jpg -*/ +*/ \ No newline at end of file diff --git a/docs/doxygen-user_fr/result_viewer.dox b/docs/doxygen-user_fr/result_viewer.dox index 6b628ff978..047ccb0845 100644 --- a/docs/doxygen-user_fr/result_viewer.dox +++ b/docs/doxygen-user_fr/result_viewer.dox @@ -1,4 +1,4 @@ -/*! \page result_viewer_page Visionneuse de résultats +/*! \page result_viewer_page Visionneuse de résultats [TOC] @@ -25,8 +25,8 @@ Ces colonnes affichent les informations suivantes:
    • (S)core - indique si l'élément est intéressant ou notable.
        -
      • Affiche une icône rouge si le fichier correspond à un hachage notable ou a été marqué avec une balise notable. -
      • Affiche une icône jaune si le fichier correspond à un élément intéressant ou a été marqué avec une balise non notable. +
      • Affiche une icône rouge si au moins un résultat d'analyse enfant est notable ou si le fichier est marqué avec une balise notable. +
      • Affiche une icône jaune si au moins un résultat d'analyse enfant est probablement notable ou si le fichier est marqué.
    • (C)omment - indique si l'élément a un commentaire dans le référentiel central ou si un commentaire est associé à une balise.
    • (O)ther occurrences - indique combien de sources de données dans le référentiel central contiennent cet élément. Le décompte comprendra l'élément sélectionné. @@ -61,4 +61,4 @@ La visionneuse "Table" peut fonctionner lentement lors de l'affichage d'un grand Vous pouvez ajuster les tailles de page via les \ref view_options_page ou désactiver complètement la pagination. -*/ +*/ \ No newline at end of file diff --git a/docs/doxygen-user_fr/tree_viewer.dox b/docs/doxygen-user_fr/tree_viewer.dox index 0f67d27271..b87973092b 100644 --- a/docs/doxygen-user_fr/tree_viewer.dox +++ b/docs/doxygen-user_fr/tree_viewer.dox @@ -1,12 +1,14 @@ -/*! \page tree_viewer_page Arborescence +/*! \page tree_viewer_page Arborescence [TOC] -L'arborescence sur le côté gauche de la fenêtre principale est l'endroit où vous pouvez parcourir les fichiers dans les sources de données du cas et trouver les résultats enregistrés à partir des analyses automatisées (Ingest). L'arborescence a cinq zones principales: +L'arborescence sur le côté gauche de la fenêtre principale est l'endroit où vous pouvez parcourir les fichiers dans les sources de données du cas et trouver les résultats enregistrés à partir des analyses automatisées (Ingest). L'arborescence a sept zones principales: - Persons / Hosts / Data Sources: Cela montre la hiérarchie arborescente de répertoires des sources de données. Vous pouvez accéder à un fichier ou à un répertoire spécifique ici. Chaque source de données ajoutée au cas est représentée sous la forme d'une sous-arborescence distincte. Si vous ajoutez une source de données plusieurs fois, elle apparaît plusieurs fois. -- Views: Des types spécifiques de fichiers provenant des sources de données sont affichés ici, agrégés par type ou par d'autres propriétés. Les fichiers ici peuvent provenir de plusieurs sources de données. -- Results: C'est ici que vous pouvez voir les résultats des analyses automatisées (Ingest) exécutée en arrière-plan ainsi que vos résultats de recherche. +- File Views: Des types spécifiques de fichiers provenant des sources de données sont affichés ici, agrégés par type ou par d'autres propriétés. Les fichiers ici peuvent provenir de plusieurs sources de données. +- Data Artifacts: C'est l'un des principaux endroits où les résultats des \ref ingest_page en cours apparaissent. +- Analysis Results: C'est l'un des autres principaux endroits où les résultats des \ref ingest_page en cours apparaissent. +- OS Accounts: C'est ici que vous pouvez voir à la fois les résultats de l'analyse automatisée exécutée en arrière-plan et les résultats de votre recherche. - Tags: C'est là que les fichiers et les résultats qui ont été \ref tagging_page "marqués" sont affichés. - Reports: Les rapports que vous avez générés ou que les modules d'acquisition ont créés s'affichent ici. @@ -43,25 +45,31 @@ L'espace non alloué (Unallocated space) représente les parties d'un système d Un exemple de l'option d'extraction de fichier unique est illustré ci-dessous. \image html extracting-unallocated-space.PNG -\section ui_tree_views Views (Vues) +\section ui_tree_views File Views (Vues des fichiers) La zone "Views" filtre tous les fichiers du cas en fonction de certaines propriétés du fichier. - File Types - Trie les fichiers par extension ou par type MIME et les affiche dans le groupe approprié. Par exemple, les fichiers avec les extensions .mp3 et .wav se retrouvent dans le groupe "Audio". - Deleted Files - Affiche les fichiers qui ont été supprimés, mais dont les noms ont été récupérés. - File Size - Trie les fichiers en fonction de leur taille. +\section ui_tree_results Data Artifacts (Artefacts de données) -\section ui_tree_results Results (Résultats) -- Extracted Content: De nombreux modules d'acquisition placeront les résultats ici: métadonnées EXIF, emplacements GPS ou historiques Web par exemple. -- Keyword Hits: Les résultats de la recherche par mot-clé s'affichent ici. -- Hashset Hits: Les résultats de la recherche de hachage s'affichent ici. -- E-Mail Messages: Les e-mails s'affichent ici. -- Interesting Items: Les éléments jugés intéressants apparaissent ici. -- Accounts: Les comptes de carte de crédit s'affichent ici. -- Tags: Tout élément que vous avez marqué apparaît ici pour que vous puissiez le retrouver facilement. +Cette section présente les artefacts de données créés en exécutant les modules d'acquisition. En général, les artefacts de données contiennent des informations concrètes extraites de la source de données. Par exemple, des journaux d'appels et des messages de journaux de communication ou des signets Web extraits d'une base de données de navigateur. + +\section ui_tree_analysis_results Analysis Results (Résultats d'analyse) + +Cette section affiche les résultats d'analyse créés en exécutant les modules d'acquisition. En général, les résultats d'analyse contiennent des informations intéressantes pour l'utilisateur. Par exemple, si l'utilisateur établit une liste de \ref hash_db_page "hachages notables", tous les hits de l'ensemble de hachage apparaîtront ici. + +\section ui_tree_os_accounts OS Accounts (Comptes de système d'exploitation) + +Cette section montre les comptes de système d'exploitation trouvés dans le cas. Voir la rubrique \ref host_os_accounts à titre d'exemple. + +\section ui_tree_tags Tags (Marquages) + +Tout élément que vous marquez s'affiche ici pour que vous puissiez le retrouver facilement. Voir la page \ref tagging_page pour plus d'informations. \section ui_tree_reports Reports (Rapports) Les rapports peuvent être ajoutés par les \subpage ingest_page ou créés en utilisant l'outil \subpage reporting_page. -*/ +*/ \ No newline at end of file diff --git a/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py b/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py index 7217584315..27f9be6161 100644 --- a/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py +++ b/pythonExamples/Aug2015DataSourceTutorial/FindContactsDb.py @@ -40,7 +40,7 @@ from java.lang import Class from java.lang import System from java.sql import DriverManager, SQLException from java.util.logging import Level -from java.util import ArrayList +from java.util import Arrays from java.io import File from org.sleuthkit.datamodel import SleuthkitCase from org.sleuthkit.datamodel import AbstractFile @@ -113,7 +113,7 @@ class ContactsDbIngestModule(DataSourceIngestModule): progressBar.switchToIndeterminate() # Use blackboard class to index blackboard artifacts for keyword search - blackboard = Case.getCurrentCase().getServices().getBlackboard() + blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard() # Find files named contacts.db, regardless of parent path fileManager = Case.getCurrentCase().getServices().getFileManager() @@ -162,30 +162,21 @@ class ContactsDbIngestModule(DataSourceIngestModule): # Make an artifact on the blackboard, TSK_CONTACT and give it attributes for each of the fields - art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT) - attributes = ArrayList() + art = file.newDataArtifact(BlackboardArtifact.Type.TSK_CONTACT, Arrays.asList( + BlackboardAttribute(BlackboardAttribute.Type.TSK_NAME_PERSON, + ContactsDbIngestModuleFactory.moduleName, name), + BlackboardAttribute(BlackboardAttribute.Type.TSK_EMAIL, + ContactsDbIngestModuleFactory.moduleName, email), + BlackboardAttribute(BlackboardAttribute.Type.TSK_PHONE_NUMBER, + ContactsDbIngestModuleFactory.moduleName, phone) + )) - attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON.getTypeID(), - ContactsDbIngestModuleFactory.moduleName, name)) - - attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL.getTypeID(), - ContactsDbIngestModuleFactory.moduleName, email)) - - attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER.getTypeID(), - ContactsDbIngestModuleFactory.moduleName, phone)) - - art.addAttributes(attributes) try: # index the artifact for keyword search - blackboard.indexArtifact(art) + blackboard.postArtifact(art, ContactsDbIngestModuleFactory.moduleName) except Blackboard.BlackboardException as e: self.log(Level.SEVERE, "Error indexing artifact " + art.getDisplayName()) - - # Fire an event to notify the UI and others that there are new artifacts - IngestServices.getInstance().fireModuleDataEvent( - ModuleDataEvent(ContactsDbIngestModuleFactory.moduleName, - BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT, None)) - + # Clean up stmt.close() dbConn.close() diff --git a/pythonExamples/Aug2015DataSourceTutorial/RunExe.py b/pythonExamples/Aug2015DataSourceTutorial/RunExe.py index 4544c1884c..230aba11f8 100644 --- a/pythonExamples/Aug2015DataSourceTutorial/RunExe.py +++ b/pythonExamples/Aug2015DataSourceTutorial/RunExe.py @@ -145,7 +145,7 @@ class RunExeIngestModule(DataSourceIngestModule): # Add each argument in its own line. I.e. "-f foo" would be two calls to .add() cmd.add(imagePaths[0]) - processBuilder = ProcessBuilder(cmd); + processBuilder = ProcessBuilder(cmd) processBuilder.redirectOutput(reportFile) ExecUtil.execute(processBuilder, DataSourceIngestModuleProcessTerminator(self.context)) diff --git a/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py b/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py index e24a8395fc..5bf710e9d5 100644 --- a/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py +++ b/pythonExamples/July2015FileTutorial_BigRound/FindBigRoundFiles.py @@ -57,6 +57,8 @@ from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.casemodule.services import Services from org.sleuthkit.autopsy.casemodule.services import FileManager from org.sleuthkit.autopsy.casemodule.services import Blackboard +from org.sleuthkit.datamodel import Score +from java.util import Arrays # Factory that defines the name and details of the module and allows Autopsy # to create instances of the modules that will do the anlaysis. @@ -107,7 +109,7 @@ class FindBigRoundFilesIngestModule(FileIngestModule): def process(self, file): # Use blackboard class to index blackboard artifacts for keyword search - blackboard = Case.getCurrentCase().getServices().getBlackboard() + blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard() # Skip non-files if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) or @@ -120,22 +122,19 @@ class FindBigRoundFilesIngestModule(FileIngestModule): # Make an artifact on the blackboard. TSK_INTERESTING_FILE_HIT is a generic type of # artifact. Refer to the developer docs for other examples. - art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT) - att = BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), - FindBigRoundFilesIngestModuleFactory.moduleName, "Big and Round Files") - art.addAttribute(att) + art = file.newAnalysisResult(BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT, Score.SCORE_LIKELY_NOTABLE, + None, "Big and Round Files", None, + Arrays.asList( + BlackboardAttribute(BlackboardAttribute.Type.TSK_SET_NAME, + FindBigRoundFilesIngestModuleFactory.moduleName, + "Big and Round Files"))).getAnalysisResult() try: - # index the artifact for keyword search - blackboard.indexArtifact(art) + # post the artifact for listeners of artifact events + blackboard.postArtifact(art, FindBigRoundFilesIngestModuleFactory.moduleName) except Blackboard.BlackboardException as e: self.log(Level.SEVERE, "Error indexing artifact " + art.getDisplayName()) - # Fire an event to notify the UI and others that there is a new artifact - IngestServices.getInstance().fireModuleDataEvent( - ModuleDataEvent(FindBigRoundFilesIngestModuleFactory.moduleName, - BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, None)) - return IngestModule.ProcessResult.OK # Where any shutdown code is run and resources are freed. diff --git a/pythonExamples/Registry_Example.py b/pythonExamples/Registry_Example.py index 885d682ab5..f99ead8086 100644 --- a/pythonExamples/Registry_Example.py +++ b/pythonExamples/Registry_Example.py @@ -45,12 +45,13 @@ from java.lang import Class from java.lang import System from java.sql import DriverManager, SQLException from java.util.logging import Level -from java.util import ArrayList +from java.util import Arrays from org.sleuthkit.datamodel import SleuthkitCase from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import ReadContentInputStream from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Blackboard from org.sleuthkit.datamodel import TskData from org.sleuthkit.autopsy.ingest import IngestModule from org.sleuthkit.autopsy.ingest.IngestModule import IngestModuleException @@ -130,12 +131,13 @@ class RegistryExampleIngestModule(DataSourceIngestModule): tempDir = os.path.join(Case.getCurrentCase().getTempDirectory(), "RegistryExample") self.log(Level.INFO, "create Directory " + tempDir) try: - os.mkdir(tempDir) + os.mkdir(tempDir) except: - self.log(Level.INFO, "ExampleRegistry Directory already exists " + tempDir) + self.log(Level.INFO, "ExampleRegistry Directory already exists " + tempDir) # Set the database to be read to the once created by the prefetch parser program - skCase = Case.getCurrentCase().getSleuthkitCase(); + skCase = Case.getCurrentCase().getSleuthkitCase() + blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard() fileManager = Case.getCurrentCase().getServices().getFileManager() # Look for files to process @@ -170,13 +172,13 @@ class RegistryExampleIngestModule(DataSourceIngestModule): # Setup Artifact and Attributes - try: - artID = skCase.addArtifactType( "TSK_REGISTRY_RUN_KEYS", "Registry Run Keys") - except: - self.log(Level.INFO, "Artifacts Creation Error, some artifacts may not exist now. ==> ") - - artId = skCase.getArtifactTypeID("TSK_REGISTRY_RUN_KEYS") - + artType = skCase.getArtifactType("TSK_REGISTRY_RUN_KEYS") + if not artType: + try: + artType = skCase.addBlackboardArtifactType( "TSK_REGISTRY_RUN_KEYS", "Registry Run Keys") + except: + self.log(Level.WARNING, "Artifacts Creation Error, some artifacts may not exist now. ==> ") + try: attributeIdRunKeyName = skCase.addArtifactAttributeType("TSK_REG_RUN_KEY_NAME", BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, "Run Key Name") except: @@ -198,25 +200,24 @@ class RegistryExampleIngestModule(DataSourceIngestModule): # RefistryKeysFound is a list that contains a list with the following records abstractFile, Registry Key Location, Key Name, Key value for registryKey in self.registryKeysFound: - attributes = ArrayList() - art = registryKey[0].newArtifact(artId) + self.log(Level.INFO, "Creating artifact for registry key with path: " + registryKey[1] + " and key: " + registryKey[2]) + art = registryKey[0].newDataArtifact(artType, Arrays.asList( + BlackboardAttribute(attributeIdRegKeyLoc, moduleName, registryKey[1]), + BlackboardAttribute(attributeIdRunKeyName, moduleName, registryKey[2]), + BlackboardAttribute(attributeIdRunKeyValue, moduleName, registryKey[3]) + )) - attributes.add(BlackboardAttribute(attributeIdRegKeyLoc, moduleName, registryKey[1])) - attributes.add(BlackboardAttribute(attributeIdRunKeyName, moduleName, registryKey[2])) - attributes.add(BlackboardAttribute(attributeIdRunKeyValue, moduleName, registryKey[3])) - art.addAttributes(attributes) - # index the artifact for keyword search try: - blackboard.indexArtifact(art) - except: - self._logger.log(Level.WARNING, "Error indexing artifact " + art.getDisplayName()) + blackboard.postArtifact(art, moduleName) + except Blackboard.BlackboardException as ex: + self.log(Level.SEVERE, "Unable to index blackboard artifact " + str(art.getArtifactTypeName()), ex) #Clean up registryExample directory and files try: - shutil.rmtree(tempDir) + shutil.rmtree(tempDir) except: - self.log(Level.INFO, "removal of directory tree failed " + tempDir) + self.log(Level.INFO, "removal of directory tree failed " + tempDir) # After all databases, post a message to the ingest messages in box. message = IngestMessage.createMessage(IngestMessage.MessageType.DATA, @@ -236,7 +237,7 @@ class RegistryExampleIngestModule(DataSourceIngestModule): softwareRegFile = RegistryHiveFile(File(softwareHive)) for runKey in self.registrySoftwareRunKeys: currentKey = self.findRegistryKey(softwareRegFile, runKey) - if len(currentKey.getValueList()) > 0: + if currentKey and len(currentKey.getValueList()) > 0: skValues = currentKey.getValueList() for skValue in skValues: regKey = [] @@ -255,7 +256,7 @@ class RegistryExampleIngestModule(DataSourceIngestModule): ntuserRegFile = RegistryHiveFile(File(ntuserHive)) for runKey in self.registryNTUserRunKeys: currentKey = self.findRegistryKey(ntuserRegFile, runKey) - if len(currentKey.getValueList()) > 0: + if currentKey and len(currentKey.getValueList()) > 0: skValues = currentKey.getValueList() for skValue in skValues: regKey = [] @@ -276,9 +277,10 @@ class RegistryExampleIngestModule(DataSourceIngestModule): for key in regKeyList: currentKey = currentKey.getSubkey(key) return currentKey - except: - # Key not found - return null + except Exception as ex: + # Key not found + self.log(Level.SEVERE, "registry key parsing issue:", ex) + return None diff --git a/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py b/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py index 5cf87cdc93..11bf862b95 100644 --- a/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py +++ b/pythonExamples/Sept2015ReportTutorial_CSV/CsvReportModule.py @@ -37,6 +37,7 @@ import os +import codecs from java.lang import System from java.util.logging import Level from org.sleuthkit.datamodel import TskData @@ -72,11 +73,11 @@ class CSVReportModule(GeneralReportModuleAdapter): # The 'baseReportDir' object being passed in is a string with the directory that reports are being stored in. Report should go into baseReportDir + getRelativeFilePath(). # The 'progressBar' object is of type ReportProgressPanel. # See: http://sleuthkit.org/autopsy/docs/api-docs/latest/classorg_1_1sleuthkit_1_1autopsy_1_1report_1_1_report_progress_panel.html - def generateReport(self, baseReportDir, progressBar): + def generateReport(self, reportSettings, progressBar): # Open the output file. - fileName = os.path.join(baseReportDir, self.getRelativeFilePath()) - report = open(fileName, 'w') + fileName = os.path.join(reportSettings.getReportDirectoryPath(), self.getRelativeFilePath()) + report = codecs.open(fileName, "w", "utf-8") # Query the database for the files (ignore the directories) sleuthkitCase = Case.getCurrentCase().getSleuthkitCase() diff --git a/pythonExamples/dataSourceIngestModule.py b/pythonExamples/dataSourceIngestModule.py index bfe745b3a4..ecb4f01477 100644 --- a/pythonExamples/dataSourceIngestModule.py +++ b/pythonExamples/dataSourceIngestModule.py @@ -53,9 +53,8 @@ from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.casemodule.services import Services from org.sleuthkit.autopsy.casemodule.services import FileManager from org.sleuthkit.autopsy.casemodule.services import Blackboard -from org.sleuthkit.autopsy.casemodule.services import Blackboard from org.sleuthkit.datamodel import Score -from java.util import ArrayList +from java.util import Arrays # Factory that defines the name and details of the module and allows Autopsy # to create instances of the modules that will do the analysis. @@ -86,8 +85,6 @@ class SampleJythonDataSourceIngestModuleFactory(IngestModuleFactoryAdapter): # Data Source-level ingest module. One gets created per data source. # TODO: Rename this to something more specific. Could just remove "Factory" from above name. class SampleJythonDataSourceIngestModule(DataSourceIngestModule): - LIKELY_NOTABLE_SCORE = Score(Score.Significance.LIKELY_NOTABLE, Score.MethodCategory.AUTO) - _logger = Logger.getLogger(SampleJythonDataSourceIngestModuleFactory.moduleName) def log(self, level, msg): @@ -118,7 +115,7 @@ class SampleJythonDataSourceIngestModule(DataSourceIngestModule): progressBar.switchToIndeterminate() # Use blackboard class to index blackboard artifacts for keyword search - blackboard = Case.getCurrentCase().getServices().getBlackboard() + blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard() # For our example, we will use FileManager to get all # files with the word "test" @@ -142,13 +139,15 @@ class SampleJythonDataSourceIngestModule(DataSourceIngestModule): # Make an artifact on the blackboard. TSK_INTERESTING_FILE_HIT is a generic type of # artfiact. Refer to the developer docs for other examples. - attrs = ArrayList() - attrs.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, SampleJythonDataSourceIngestModuleFactory.moduleName, "Test file")) - art = file.newAnalysisResult(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, self.LIKELY_NOTABLE_SCORE, None, "Test file", None, attrs) + attrs = Arrays.asList(BlackboardAttribute(BlackboardAttribute.Type.TSK_SET_NAME, + SampleJythonDataSourceIngestModuleFactory.moduleName, + "Test file")) + art = file.newAnalysisResult(BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT, Score.SCORE_LIKELY_NOTABLE, + None, "Test file", None, attrs).getAnalysisResult() try: - # index the artifact for keyword search - blackboard.indexArtifact(art) + # post the artifact for listeners of artifact events. + blackboard.postArtifact(art, SampleJythonDataSourceIngestModuleFactory.moduleName) except Blackboard.BlackboardException as e: self.log(Level.SEVERE, "Error indexing artifact " + art.getDisplayName()) diff --git a/pythonExamples/fileIngestModule.py b/pythonExamples/fileIngestModule.py index e4aa12bab7..f72f1c4792 100644 --- a/pythonExamples/fileIngestModule.py +++ b/pythonExamples/fileIngestModule.py @@ -55,8 +55,7 @@ from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.casemodule.services import Services from org.sleuthkit.autopsy.casemodule.services import FileManager from org.sleuthkit.autopsy.casemodule.services import Blackboard -from org.sleuthkit.datamodel import Score -from java.util import ArrayList +from java.util import Arrays # Factory that defines the name and details of the module and allows Autopsy # to create instances of the modules that will do the anlaysis. @@ -89,7 +88,6 @@ class SampleJythonFileIngestModuleFactory(IngestModuleFactoryAdapter): # TODO: Rename this to something more specific. Could just remove "Factory" from above name. # Looks at the attributes of the passed in file. class SampleJythonFileIngestModule(FileIngestModule): - LIKELY_NOTABLE_SCORE = Score(Score.Significance.LIKELY_NOTABLE, Score.MethodCategory.AUTO) _logger = Logger.getLogger(SampleJythonFileIngestModuleFactory.moduleName) @@ -119,7 +117,7 @@ class SampleJythonFileIngestModule(FileIngestModule): return IngestModule.ProcessResult.OK # Use blackboard class to index blackboard artifacts for keyword search - blackboard = Case.getCurrentCase().getServices().getBlackboard() + blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard() # For an example, we will flag files with .txt in the name and make a blackboard artifact. if file.getName().lower().endswith(".txt"): @@ -129,23 +127,18 @@ class SampleJythonFileIngestModule(FileIngestModule): # Make an artifact on the blackboard. TSK_INTERESTING_FILE_HIT is a generic type of # artifact. Refer to the developer docs for other examples. - attrs = ArrayList() - attrs.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, + attrs = Arrays.asList(BlackboardAttribute(BlackboardAttribute.Type.TSK_SET_NAME, SampleJythonFileIngestModuleFactory.moduleName, "Text Files")) - art = file.newAnalysisResult(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, self.LIKELY_NOTABLE_SCORE, None, "Text Files", None, attrs) + art = file.newAnalysisResult(BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT, Score.SCORE_LIKELY_NOTABLE, + None, "Text Files", None, attrs).getAnalysisResult() try: - # index the artifact for keyword search - blackboard.indexArtifact(art) + # post the artifact for listeners of artifact events + blackboard.postArtifact(art, SampleJythonFileIngestModuleFactory.moduleName) except Blackboard.BlackboardException as e: self.log(Level.SEVERE, "Error indexing artifact " + art.getDisplayName()) - # Fire an event to notify the UI and others that there is a new artifact - IngestServices.getInstance().fireModuleDataEvent( - ModuleDataEvent(SampleJythonFileIngestModuleFactory.moduleName, - BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, None)) - # For the example (this wouldn't be needed normally), we'll query the blackboard for data that was added # by other modules. We then iterate over its attributes. We'll just print them, but you would probably # want to do something with them. diff --git a/pythonExamples/reportmodule.py b/pythonExamples/reportmodule.py index 33c6941eda..df973e05fb 100644 --- a/pythonExamples/reportmodule.py +++ b/pythonExamples/reportmodule.py @@ -67,10 +67,12 @@ class SampleGeneralReportModule(GeneralReportModuleAdapter): return "sampleReport.txt" # TODO: Update this method to make a report - # The 'baseReportDir' object being passed in is a string with the directory that reports are being stored in. Report should go into baseReportDir + getRelativeFilePath(). + # The 'reportSettings' object being passed in is an instance of org.sleuthkit.autopsy.report.GeneralReportSettings. + # GeneralReportSettings.getReportDirectoryPath() is the directory that reports are being stored in. + # Report should go into GeneralReportSettings.getReportDirectoryPath() + getRelativeFilePath(). # The 'progressBar' object is of type ReportProgressPanel. # See: http://sleuthkit.org/autopsy/docs/api-docs/latest/classorg_1_1sleuthkit_1_1autopsy_1_1report_1_1_report_progress_panel.html - def generateReport(self, baseReportDir, progressBar): + def generateReport(self, reportSettings, progressBar): # For an example, we write a file with the number of files created in the past 2 weeks # Configure progress bar for 2 tasks @@ -95,7 +97,7 @@ class SampleGeneralReportModule(GeneralReportModuleAdapter): progressBar.increment() # Write the count to the report file. - fileName = os.path.join(baseReportDir, self.getRelativeFilePath()) + fileName = os.path.join(reportSettings.getReportDirectoryPath(), self.getRelativeFilePath()) report = open(fileName, 'w') report.write("file count = %d" % fileCount) report.close() diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index fc9498d4c6..f0aae2c2a5 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -452,7 +452,8 @@ class TskGuidUtils: guid_vs_info = TskGuidUtils._get_guid_dict(db_conn, "SELECT obj_id, vs_type, img_offset FROM tsk_vs_info", "_") guid_fs_info = TskGuidUtils._get_guid_dict(db_conn, "SELECT obj_id, img_offset, fs_type FROM tsk_fs_info", "_") guid_image_names = TskGuidUtils._get_guid_dict(db_conn, "SELECT obj_id, name FROM tsk_image_names " - "WHERE sequence=0") + "WHERE sequence=0", + normalizer=get_filename) guid_os_accounts = TskGuidUtils._get_guid_dict(db_conn, "SELECT os_account_obj_id, addr FROM tsk_os_accounts") guid_reports = TskGuidUtils._get_guid_dict(db_conn, "SELECT obj_id, path FROM reports", normalizer=normalize_file_path) @@ -625,6 +626,22 @@ def get_path_segs(path: Union[str, None]) -> Union[List[str], None]: return None +def get_filename(path: Union[str, None]) -> Union[str, None]: + """ + Returns the last segment of a file path. + Args: + path: The path. + + Returns: The last segment of the path + + """ + path_segs = get_path_segs(path) + if path_segs is not None and len(path_segs) > 0: + return path_segs[-1] + else: + return None + + def index_of(lst, search_item) -> int: """ Returns the index of the item in the list or -1. @@ -827,7 +844,9 @@ def normalize_tsk_event_descriptions(guid_util: TskGuidUtils, row: Dict[str, any # replace object ids with information that is deterministic row_copy['event_description_id'] = MASKED_ID row_copy['content_obj_id'] = guid_util.get_guid_for_file_objid(row['content_obj_id']) - row_copy['artifact_id'] = guid_util.get_guid_for_artifactid(row['artifact_id']) if row['artifact_id'] else None + row_copy['artifact_id'] = guid_util.get_guid_for_artifactid(row['artifact_id']) \ + if row['artifact_id'] is not None else None + row_copy['data_source_obj_id'] = guid_util.get_guid_for_file_objid(row['data_source_obj_id']) if row['full_description'] == row['med_description'] == row['short_description']: row_copy['full_description'] = _mask_event_desc(row['full_description']) @@ -853,8 +872,8 @@ def normalize_ingest_jobs(guid_util: TskGuidUtils, row: Dict[str, any]) -> Dict[ start_time = row['start_date_time'] end_time = row['end_date_time'] if start_time <= end_time: - row_copy['start_date_time'] = 0 - row_copy['end_date_time'] = 0 + row_copy['start_date_time'] = MASKED_TIME + row_copy['end_date_time'] = MASKED_TIME return row_copy @@ -916,6 +935,7 @@ def normalize_tsk_files(guid_util: TskGuidUtils, row: Dict[str, any]) -> Dict[st row_copy['md5'] = "MD5_IGNORED" row_copy['sha256'] = "SHA256_IGNORED" + row_copy['data_source_obj_id'] = guid_util.get_guid_for_file_objid(row['data_source_obj_id']) row_copy['obj_id'] = MASKED_OBJ_ID row_copy['os_account_obj_id'] = 'MASKED_OS_ACCOUNT_OBJ_ID' row_copy['parent_path'] = normalize_file_path(row['parent_path']) @@ -1009,6 +1029,7 @@ def normalize_tsk_objects(guid_util: TskGuidUtils, row: Dict[str, any]) -> Dict[ return row_copy +MASKED_TIME = "MASKED_TIME" MASKED_OBJ_ID = "MASKED_OBJ_ID" MASKED_ID = "MASKED_ID" @@ -1027,14 +1048,15 @@ TABLE_NORMALIZATIONS: Dict[str, TableNormalization] = { "added_date_time": "{dateTime}" }), "image_gallery_groups": NormalizeColumns({ - "group_id": MASKED_ID + "group_id": MASKED_ID, + "data_source_obj_id": lambda guid_util, col: guid_util.get_guid_for_objid(col, omitted_value=None), }), "image_gallery_groups_seen": IGNORE_TABLE, "ingest_jobs": NormalizeRow(normalize_ingest_jobs), "reports": NormalizeColumns({ "obj_id": MASKED_OBJ_ID, "path": "AutopsyTestCase", - "crtime": 0 + "crtime": MASKED_TIME }), "tsk_aggregate_score": NormalizeColumns({ "obj_id": lambda guid_util, col: guid_util.get_guid_for_objid(col, omitted_value="Object ID Omitted"), @@ -1053,8 +1075,7 @@ TABLE_NORMALIZATIONS: Dict[str, TableNormalization] = { "tsk_event_descriptions": NormalizeRow(normalize_tsk_event_descriptions), "tsk_events": NormalizeColumns({ "event_id": "MASKED_EVENT_ID", - "event_description_id": None, - "time": None, + "event_description_id": 'ID OMITTED' }), "tsk_examiners": NormalizeColumns({ "login_name": "{examiner_name}" @@ -1064,6 +1085,9 @@ TABLE_NORMALIZATIONS: Dict[str, TableNormalization] = { "obj_id": lambda guid_util, col: guid_util.get_guid_for_file_objid(col) }), "tsk_files_path": NormalizeRow(normalize_tsk_files_path), + "tsk_image_names": NormalizeColumns({ + "name": lambda guid_util, col: get_filename(col) + }), "tsk_objects": NormalizeRow(normalize_tsk_objects), "tsk_os_account_attributes": NormalizeColumns({ "id": MASKED_ID, @@ -1121,7 +1145,8 @@ def write_normalized(guid_utils: TskGuidUtils, output_file, db_conn, table: str, # show row as json-like value entries = [] for column in column_names: - value = get_sql_insert_value(row_dict[column] if column in row_dict and row_dict[column] else None) + dict_value = row_dict[column] if column in row_dict and row_dict[column] is not None else None + value = get_sql_insert_value(dict_value) if value is not None: entries.append((column, value)) insert_values = ", ".join([f"{pr[0]}: {pr[1]}" for pr in entries])