Merge pull request #3398 from esaunders/report_index_and_search

Added support for Report indexing

Core/src/org/sleuthkit/autopsy/casemodule/Case.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2017 Basis Technology Corp.
+ * Copyright 2011-2018 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -998,17 +998,16 @@ public class Case {
         }
     }
 
-    private static String getNameForTitle(){
+    private static String getNameForTitle() {
         //Method should become unnecessary once technical debt story 3334 is done.
-        if (UserPreferences.getAppName().equals(Version.getName())){
+        if (UserPreferences.getAppName().equals(Version.getName())) {
             //Available version number is version number for this application
             return String.format("%s %s", UserPreferences.getAppName(), Version.getVersion());
-        }
+        } else {
-        else {
             return UserPreferences.getAppName();
         }
     }
-    
+
     /**
      * Update the GUI to to reflect the current case.
      */
@@ -1526,7 +1525,25 @@ public class Case {
      * @throws TskCoreException if there is a problem adding the report to the
      *                          case database.
      */
-    public void addReport(String localPath, String srcModuleName, String reportName) throws TskCoreException {
+    public Report addReport(String localPath, String srcModuleName, String reportName) throws TskCoreException {
+        return addReport(localPath, srcModuleName, reportName, null);
+    }
+
+    /**
+     * Adds a report to the case.
+     *
+     * @param localPath     The path of the report file, must be in the case
+     *                      directory or one of its subdirectories.
+     * @param srcModuleName The name of the module that created the report.
+     * @param reportName    The report name, may be empty.
+     * @param source        The Content used to create the report, may be empty.
+     *
+     * @return The new Report instance.
+     *
+     * @throws TskCoreException if there is a problem adding the report to the
+     *                          case database.
+     */
+    public Report addReport(String localPath, String srcModuleName, String reportName, Content source) throws TskCoreException {
         String normalizedLocalPath;
         try {
             normalizedLocalPath = Paths.get(localPath).normalize().toString();
@@ -1536,6 +1553,7 @@ public class Case {
         }
         Report report = this.caseDb.addReport(normalizedLocalPath, srcModuleName, reportName);
         eventPublisher.publish(new ReportAddedEvent(report));
+        return report;
     }
 
     /**
@@ -1616,13 +1634,14 @@ public class Case {
     /**
      * Constructs a Case object for a new Autopsy case.
      *
-     * @param caseType        The type of case (single-user or multi-user).
+     * @param caseType    The type of case (single-user or multi-user).
-     * @param caseDir         The full path of the case directory. The directory
+     * @param caseDir     The full path of the case directory. The directory
-     *                        will be created if it doesn't already exist; if it
+     *                    will be created if it doesn't already exist; if it
-     *                        exists, it is ASSUMED it was created by calling
+     *                    exists, it is ASSUMED it was created by calling
-     *                        createCaseDirectory.
+     *                    createCaseDirectory.
-     * @param caseDetails     Contains details of the case, such as examiner, display name, etc
+     * @param caseDetails Contains details of the case, such as examiner,
-     *                        
+     *                    display name, etc
+     *
      */
     private Case(CaseType caseType, String caseDir, CaseDetails caseDetails) {
         metadata = new CaseMetadata(caseType, caseDir, displayNameToUniqueName(caseDetails.getCaseDisplayName()), caseDetails);

Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2017 Basis Technology Corp.
+ * Copyright 2011-2018 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearchservice;
 import java.io.Closeable;
 import org.sleuthkit.autopsy.casemodule.CaseMetadata;
 import org.sleuthkit.datamodel.BlackboardArtifact;
+import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
@@ -50,6 +51,15 @@ public interface KeywordSearchService extends Closeable {
      */
     public void indexArtifact(BlackboardArtifact artifact) throws TskCoreException;
 
+    /**
+     * Add the given Content object to the text index.
+     *
+     * @param content The content to index.
+     *
+     * @throws TskCoreException
+     */
+    public void index(Content content) throws TskCoreException;
+
     /**
      * Deletes the keyword search text index for a case.
      *

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2017 Basis Technology Corp.
+ * Copyright 2011-2018 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,12 +31,14 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.keywordsearch.Chunker.Chunk;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.BlackboardArtifact;
+import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.DerivedFile;
 import org.sleuthkit.datamodel.Directory;
 import org.sleuthkit.datamodel.File;
 import org.sleuthkit.datamodel.LayoutFile;
 import org.sleuthkit.datamodel.LocalDirectory;
 import org.sleuthkit.datamodel.LocalFile;
+import org.sleuthkit.datamodel.Report;
 import org.sleuthkit.datamodel.SlackFile;
 import org.sleuthkit.datamodel.SleuthkitItemVisitor;
 import org.sleuthkit.datamodel.SleuthkitVisitableItem;
@@ -361,6 +363,31 @@ class Ingester {
             }
             return params;
         }
+
+        /**
+         * Get the field map for artifacts.
+         *
+         * @param report The report to get fields for.
+         *
+         * @return The field map for the given report.
+         */
+        @Override
+        public Map<String, String> visit(Report report) {
+            Map<String, String> params = new HashMap<>();
+            params.put(Server.Schema.ID.toString(), Long.toString(report.getId()));
+            try {
+                Content dataSource = report.getDataSource();
+                if (null == dataSource) {
+                    params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1));
+                } else {
+                    params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(dataSource.getId()));
+                }
+            } catch (TskCoreException ex) {
+                logger.log(Level.SEVERE, "Could not get data source id to properly index the report, using default value. Id: " + report.getId(), ex); //NON-NLS
+                params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1));
+            }
+            return params;
+        }
     }
 
     /**

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2017 Basis Technology Corp.
+ * Copyright 2011-2018 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -46,6 +46,7 @@ import org.sleuthkit.autopsy.progress.ProgressIndicator;
 import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
 import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
 import org.sleuthkit.datamodel.BlackboardArtifact;
+import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
@@ -95,6 +96,35 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
         }
     }
 
+    /**
+     * Add the given Content object to the text index.
+     * @param content The content to index.
+     * @throws TskCoreException 
+     */
+    @Override
+    public void index(Content content) throws TskCoreException {
+        if (content == null) {
+            return;
+        }
+        final Ingester ingester = Ingester.getDefault();
+
+        try {
+            ingester.indexText(new TikaTextExtractor(), content, null);
+        } catch (Ingester.IngesterException ex) {
+            try {
+                // Try the StringsTextExtractor if Tika extractions fails.
+                ingester.indexText(new StringsTextExtractor(), content, null);
+            } catch (Ingester.IngesterException ex1) {
+                throw new TskCoreException(ex.getCause().getMessage(), ex1);
+            }        
+        }
+        
+        // TODO: Review whether this is the right thing to do. We typically use
+        // a combination of autoCommit and the SearchRunner to ensure that data
+        // is committed but that might not be sufficient for reports (or artifacts).
+        ingester.commit();
+    }
+    
     /**
      * Tries to connect to the keyword search service.
      *

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TikaTextExtractor.java

@@ -40,7 +40,6 @@ import org.apache.tika.parser.ParsingReader;
 import org.apache.tika.parser.microsoft.OfficeParserConfig;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.ReadContentInputStream;
 

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java

@@ -2,7 +2,7 @@
  *
  * Autopsy Forensic Browser
  *
- * Copyright 2012-2014 Basis Technology Corp.
+ * Copyright 2012-2018 Basis Technology Corp.
  *
  * Copyright 2012 42six Solutions.
  * Contact: aebadirad <at> 42six <dot> com
@@ -50,9 +50,11 @@ import org.w3c.dom.NodeList;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
 import java.nio.file.Path;
+import org.openide.util.Lookup;
 import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
+import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
 
 /**
  * Extract windows registry data using regripper. Runs two versions of
@@ -202,7 +204,17 @@ class ExtractRegistry extends Extract {
             // create a report for the full output
             if (!regOutputFiles.fullPlugins.isEmpty()) {
                 try {
-                    currentCase.addReport(regOutputFiles.fullPlugins, NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace"), "RegRipper " + regFile.getUniquePath()); //NON-NLS
+                    Report report = currentCase.addReport(regOutputFiles.fullPlugins,
+                            NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace"),
+                            "RegRipper " + regFile.getUniquePath(), regFile); //NON-NLS
+
+                    // Index the report content so that it will be available for keyword search.
+                    KeywordSearchService searchService = Lookup.getDefault().lookup(KeywordSearchService.class);
+                    if (null == searchService) {
+                        logger.log(Level.WARNING, "Keyword search service not found. Report will not be indexed");
+                    } else {
+                        searchService.index(report);
+                    }
                 } catch (TskCoreException e) {
                     this.addErrorMessage("Error adding regripper output as Autopsy report: " + e.getLocalizedMessage()); //NON-NLS
                 }
@@ -413,12 +425,14 @@ class ExtractRegistry extends Extract {
                                             installtime = Long.valueOf(Tempdate) / 1000;
                                         } catch (ParseException e) {
                                             logger.log(Level.SEVERE, "RegRipper::Conversion on DateTime -> ", e); //NON-NLS
-                                        }   break;
+                                        }
+                                        break;
                                     default:
                                         break;
                                 }
                             }
-                        }   try {
+                        }
+                        try {
                             Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
                             bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, parentModuleName, version));
                             if (installtime != null) {
@@ -475,7 +489,8 @@ class ExtractRegistry extends Extract {
                                         break;
                                 }
                             }
-                        }   try {
+                        }
+                        try {
                             Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
                             bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VERSION, parentModuleName, os));
                             bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROCESSOR_ARCHITECTURE, parentModuleName, procArch));
@@ -514,7 +529,8 @@ class ExtractRegistry extends Extract {
                                     domain = value;
                                 }
                             }
-                        }   try {
+                        }
+                        try {
                             Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
                             bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, parentModuleName, compName));
                             bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, parentModuleName, domain));
@@ -656,7 +672,7 @@ class ExtractRegistry extends Extract {
                                                     parentModuleName, sid));
                                             bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
                                                     parentModuleName, homeDir));
-                                            
+
                                             bbart.addAttributes(bbattributes);
                                             // index the artifact for keyword search
                                             this.indexArtifact(bbart);