Merge pull request #7643 from sleuthkit/develop

develop into new_table_load (7/12/22)

Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java

@@ -75,6 +75,10 @@ public class CommandLineIngestManager extends CommandLineManager {
     private Case caseForJob = null;
     private AutoIngestDataSource dataSource = null;
 
+    static int CL_SUCCESS = 0;
+    static int CL_RUN_FAILURE = -1;
+    static int CL_PROCESS_FAILURE = 1;
+
     public CommandLineIngestManager() {
     }
 
@@ -82,7 +86,11 @@ public class CommandLineIngestManager extends CommandLineManager {
         new Thread(new JobProcessingTask()).start();
     }
 
-    public void stop() {
+    void stop() {
+        stop(CL_SUCCESS);
+    }
+
+    void stop(int errorCode) {
         try {
             // close current case if there is one open
             Case.closeCurrentCase();
@@ -91,7 +99,11 @@ public class CommandLineIngestManager extends CommandLineManager {
         }
 
         // shut down Autopsy
+        if (errorCode == CL_SUCCESS) {
             LifecycleManager.getDefault().exit();
+        } else {
+            LifecycleManager.getDefault().exit(errorCode);
+        }
     }
 
     private final class JobProcessingTask implements Runnable {
@@ -115,6 +127,7 @@ public class CommandLineIngestManager extends CommandLineManager {
         @Override
         public void run() {
             LOGGER.log(Level.INFO, "Job processing task started");
+            int errorCode = CL_SUCCESS;
 
             try {
                 // read command line inputs
@@ -132,14 +145,14 @@ public class CommandLineIngestManager extends CommandLineManager {
                         commands = ((CommandLineOptionProcessor) processor).getCommands();
                     }
                 }
-
+                try {
                     if (commands == null || commands.isEmpty()) {
                         LOGGER.log(Level.SEVERE, "No command line commands specified");
                         System.out.println("No command line commands specified");
+                        errorCode = CL_RUN_FAILURE;
                         return;
                     }
 
-                try {
                     // Commands are already stored in order in which they should be executed                
                     for (CommandLineCommand command : commands) {
                         CommandLineCommand.CommandType type = command.getType();
@@ -165,6 +178,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                                 LOGGER.log(Level.SEVERE, "Error creating or opening case " + baseCaseName, ex);
                                 System.out.println("Error creating or opening case " + baseCaseName);
                                 // Do not process any other commands
+                                errorCode = CL_RUN_FAILURE;
                                 return;
                             }
                             break;
@@ -193,6 +207,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                                 LOGGER.log(Level.SEVERE, "Error adding data source " + dataSourcePath, ex);
                                 System.out.println("Error adding data source " + dataSourcePath);
                                 // Do not process any other commands
+                                errorCode = CL_RUN_FAILURE;
                                 return;
                             }
                             break;
@@ -224,6 +239,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                                         LOGGER.log(Level.SEVERE, "Exception while trying to find data source with object ID " + dataSourceId, ex);
                                         System.out.println("Exception while trying to find data source with object ID " + dataSourceId);
                                         // Do not process any other commands
+                                        errorCode = CL_RUN_FAILURE;
                                         return;
                                     }
 
@@ -249,6 +265,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                                 LOGGER.log(Level.SEVERE, "Error running ingest on data source " + dataSourcePath, ex);
                                 System.out.println("Error running ingest on data source " + dataSourcePath);
                                 // Do not process any other commands
+                                errorCode = CL_RUN_FAILURE;
                                 return;
                             }
                             break;
@@ -275,6 +292,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                                 String msg = "Error opening case " + baseCaseName + " in directory: " + rootOutputDirectory;
                                 LOGGER.log(Level.SEVERE, msg, ex);
                                 System.out.println(msg);
+                                errorCode = CL_RUN_FAILURE;
                                 // Do not process any other commands
                                 return;
                             }
@@ -309,6 +327,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                                 String msg = "Error opening case " + baseCaseName + " in directory: " + rootOutputDirectory;
                                 LOGGER.log(Level.SEVERE, msg, ex);
                                 System.out.println(msg);
+                                errorCode = CL_RUN_FAILURE;
                                 // Do not process any other commands
                                 return;
                             }
@@ -317,7 +336,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                                 List<IngestProfile> profiles = IngestProfiles.getIngestProfiles();
                                 GsonBuilder gb = new GsonBuilder();
                                 System.out.println("Listing ingest profiles");
-                                for(IngestProfile profile: profiles) {
+                                for (IngestProfile profile : profiles) {
                                     String jsonText = gb.create().toJson(profile);
                                     System.out.println(jsonText);
                                 }
@@ -337,7 +356,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                      */
                     LOGGER.log(Level.SEVERE, "Unexpected error", ex);
                     System.out.println("Unexpected error. Exiting...");
-
+                    errorCode = CL_RUN_FAILURE;
                 } finally {
                     try {
                         Case.closeCurrentCase();
@@ -352,7 +371,7 @@ public class CommandLineIngestManager extends CommandLineManager {
                 System.out.println("Job processing task finished");
 
                 // shut down Autopsy
-                stop();
+                stop(errorCode);
             }
         }
 
@@ -366,8 +385,11 @@ public class CommandLineIngestManager extends CommandLineManager {
          * @throws AutoIngestDataSourceProcessorException if there was a DSP
          *                                                processing error.
          *
-         * @throws InterruptedException running the job processing task while
+         * @throws InterruptedException                   running the job
-         * blocking, i.e., if auto ingest is shutting down.
+         *                                                processing task while
+         *                                                blocking, i.e., if
+         *                                                auto ingest is
+         *                                                shutting down.
          */
         private void runDataSourceProcessor(Case caseForJob, AutoIngestDataSource dataSource) throws InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException {
 
@@ -473,9 +495,10 @@ public class CommandLineIngestManager extends CommandLineManager {
          *
          * @throws AnalysisStartupException if there is an error analyzing the
          *                                  data source.
-         * @throws InterruptedException if the thread running the job processing
+         * @throws InterruptedException     if the thread running the job
-         * task is interrupted while blocked, i.e., if auto ingest is shutting
+         *                                  processing task is interrupted while
-         * down.
+         *                                  blocked, i.e., if auto ingest is
+         *                                  shutting down.
          */
         private void analyze(AutoIngestDataSource dataSource, String ingestProfileName) throws AnalysisStartupException, InterruptedException {
 
@@ -523,11 +546,12 @@ public class CommandLineIngestManager extends CommandLineManager {
                         IngestJob ingestJob = ingestJobStartResult.getJob();
                         if (null != ingestJob) {
                             /*
-                             * Block until notified by the ingest job event listener
+                             * Block until notified by the ingest job event
-                             * or until interrupted because auto ingest is shutting
+                             * listener or until interrupted because auto ingest
-                             * down. For very small jobs, it is possible that ingest has
+                             * is shutting down. For very small jobs, it is
-                             * completed by the time we get here, so check periodically
+                             * possible that ingest has completed by the time we
-                             * in case the event was missed.
+                             * get here, so check periodically in case the event
+                             * was missed.
                              */
                             while (IngestManager.getInstance().isIngestRunning()) {
                                 ingestLock.wait(60000);  // Check every minute

Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java

@@ -366,7 +366,7 @@ public class CommandLineOptionProcessor extends OptionProcessor {
      */
     private void handleError(String errorMessage) throws CommandException {
         logger.log(Level.SEVERE, errorMessage);
-        throw new CommandException(1, errorMessage);
+        throw new CommandException(CommandLineIngestManager.CL_PROCESS_FAILURE, errorMessage);
     }
     
     public void addPropertyChangeListener(

docs/doxygen-user/command_line_ingest.dox

@@ -45,7 +45,7 @@ The table below shows a summary of the command line operations. You can run one
 --caseType (optional)</pre></td><td><pre>--createCase --caseName="test5" --caseBaseDir="C:\work\cases"
 --createCase --caseName="test_multi" --caseBaseDir="\\WIN-2913\work\cases" --caseType="multi"</pre></td></tr>
 
-<tr><td><b>Open Existing Case</b></td><td>&nbsp;</td><td><pre>--caseDir</pre></td><td><pre>--caseDir="C:\work\Cases\test5_2019_09_20_11_01_29"</pre></td></tr>
+<tr><td><b>Open Existing Case</b></td><td>&nbsp;</td><td><pre>--caseBaseDir</pre></td><td><pre>--caseBaseDir="C:\work\Cases"</pre></td></tr>
 
 <tr><td><b>Add a Data Source</b></td><td><pre>--addDataSource
 --runIngest (optional)
@@ -60,6 +60,7 @@ The table below shows a summary of the command line operations. You can run one
 --generateReports="kmlReport"</pre></td></tr>
 
 <tr><td><b>Create List of Data Sources</b></td><td><pre>--listAllDataSources</pre></td><td>&nbsp;</td><td></td><pre>--listAllDataSources</pre></tr>
+<tr><td><b>Create List Ingest Profiles</b></td><td><pre>--listAllIngestProfiles</pre></td><td>&nbsp;</td><td></td><pre>--listAllIngestProfiles</pre></tr>
 </table>
 
 
@@ -86,7 +87,7 @@ autopsy64.exe --createCase --caseName="test_multi" --caseBaseDir="\\WIN-2913\wor
 Once a case is created you will need to use the full path to the case instead of the case name and base folder. For example, if we created the empty case "test5" as above, we could use the following command to add a data source to it:
 
 \verbatim
-autopsy64.exe --caseDir="C:\work\Cases\test5_2019_09_20_11_01_29" --addDataSource 
+autopsy64.exe --caseName="test5" --caseBaseDir="C:\work\Cases" --addDataSource 
    --dataSourcePath="R:\work\images\small2.img"
 \endverbatim
 
@@ -106,14 +107,14 @@ autopsy64.exe --createCase --caseName="test6" --caseBaseDir="C:\work\cases" --ad
 And here we'll add another data source ("green_images.img") to the case we just made and run ingest on it. Note that ingest will only run on the new data source ("green_images.img"), not the one already in the case ("blue_images.img").
 
 \verbatim
-autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --addDataSource --runIngest 
+autopsy64.exe --caseName="test6" --caseBaseDir="C:\work\cases" --addDataSource --runIngest 
    --dataSourcePath="R:\work\images\green_images.img"
 \endverbatim
 
 Next we'll add a third data source ("red_images.img") to the case and run ingest using a custom ingest profile "imageAnalysis" created as described in the \ref command_line_ingest_profile "Configuring Ingest Profiles" section above. 
 
 \verbatim
-autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --addDataSource --runIngest="imageAnalysis"
+autopsy64.exe --caseName="test6" --caseBaseDir="C:\work\cases" --addDataSource --runIngest="imageAnalysis"
    --dataSourcePath="R:\work\images\red_images.img"
 \endverbatim
 
@@ -145,7 +146,7 @@ If you've run with the --listAllDataSources option, there will be at least one f
 You can also look through the addDataSource files to find the one corresponding to the file you want to ingest. The format will be the same. Once you know the data source object ID, you can use the --dataSourceObjectId option to specify it. For example, this will run ingest on "blue_images.img":
 
 \verbatim
-autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --runIngest --dataSourceObjectId=1
+autopsy64.exe --caseName="test6" --caseBaseDir="C:\work\cases" --runIngest --dataSourceObjectId=1
 \endverbatim
 
 \subsection command_line_report Generating Reports
@@ -153,14 +154,14 @@ autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --runIngest --
 You can generate a report on the case using the --generateReports option. You can select which report type to export through the Autopsy options panel (see the \ref command_line_ingest_config "configuration section"). This option can be run alone or at the same time as you're processing a data source. In this example we're adding a new data source ("small2.img") and generating a report.
 
 \verbatim
-autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --addDataSource
+autopsy64.exe --caseName="test6" --caseBaseDir="C:\work\cases" --addDataSource
    --dataSourcePath="R:\work\images\small2.img" --runIngest --generateReports
 \endverbatim
 
 The example above uses the default report profile. If you set up a custom report profile as described in the \ref command_line_report_profile "Configuring Ingest Profiles section" above, you can specify that profile after the --generateReports option.
 
 \verbatim
-autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --generateReports="html"
+autopsy64.exe --caseName="test6" --caseBaseDir="C:\work\cases" --generateReports="html"
 \endverbatim
 
 \subsection command_line_listds Listing All Data Sources
@@ -168,9 +169,33 @@ autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --generateRepo
 You can add the --listAllDataSources at any time to output a list of all data sources currently in the case along with their object IDs, to be used when \ref command_line_existing_ds "running on an existing data source". This command can even be run alone with just the path to the case.
 
 \verbatim
-autopsy64.exe --caseDir="C:\work\cases\test6_2019_09_20_13_00_51" --listAllDataSources
+autopsy64.exe --caseName="test6" --caseBaseDir="C:\work\cases\" --listAllDataSources
 \endverbatim
 
+\subsection command_line_listip Listing All Ingest Profiles
+
+You can add the --listAllIngestProfiles at any time to output a list of all ingest profiles. This command can be run alone without other options.
+
+\verbatim
+autopsy64.exe --listAllIngestProfiles
+\endverbatim
+
+If there are ingest profiles the output will be similar to:
+
+\verbatim
+Listing ingest profiles
+{
+  "name" : "IngestProfile1",
+  "description" : "This is a description of IngestProfile1",
+  "fileIngestFilter" : "All Files, Directories, and Unallocated Space"
+} 
+{
+  "name" : "IngestProfile2",
+  "description" : "This is a description of IngestProfile2",
+  "fileIngestFilter" : "All Files, Directories, and Unallocated Space"
+}
+Ingest profile list complete
+\endverbatim
 
 \section command_line_ingest_run Running Autopsy
 

pythonExamples/dataSourceIngestModule.py

@@ -137,12 +137,12 @@ class SampleJythonDataSourceIngestModule(DataSourceIngestModule):
             self.log(Level.INFO, "Processing file: " + file.getName())
             fileCount += 1
 
-            # Make an artifact on the blackboard.  TSK_INTERESTING_ITEM is a generic type of
+            # Make an artifact on the blackboard.  TSK_INTERESTING_FILE_HIT is a generic type of
             # artifact.  Refer to the developer docs for other examples.
             attrs = Arrays.asList(BlackboardAttribute(BlackboardAttribute.Type.TSK_SET_NAME,
                                                       SampleJythonDataSourceIngestModuleFactory.moduleName,
                                                       "Test file"))
-            art = file.newAnalysisResult(BlackboardArtifact.Type.TSK_INTERESTING_ITEM, Score.SCORE_LIKELY_NOTABLE,
+            art = file.newAnalysisResult(BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT, Score.SCORE_LIKELY_NOTABLE,
                                          None, "Test file", None, attrs).getAnalysisResult()
 
             try:

pythonExamples/fileIngestModule.py

@@ -129,12 +129,12 @@ class SampleJythonFileIngestModule(FileIngestModule):
             self.log(Level.INFO, "Found a text file: " + file.getName())
             self.filesFound+=1
 
-            # Make an artifact on the blackboard.  TSK_INTERESTING_ITEM is a generic type of
+            # Make an artifact on the blackboard.  TSK_INTERESTING_FILE_HIT is a generic type of
             # artifact.  Refer to the developer docs for other examples.
             attrs = Arrays.asList(BlackboardAttribute(BlackboardAttribute.Type.TSK_SET_NAME,
                   SampleJythonFileIngestModuleFactory.moduleName, "Text Files"))
 
-            art = file.newAnalysisResult(BlackboardArtifact.Type.TSK_INTERESTING_ITEM, Score.SCORE_LIKELY_NOTABLE,
+            art = file.newAnalysisResult(BlackboardArtifact.Type.TSK_INTERESTING_FILE_HIT, Score.SCORE_LIKELY_NOTABLE,
                                          None, "Text Files", None, attrs).getAnalysisResult()
 
             try: