Merge pull request #2490 from eugene7646/progress_and_cancellation

Support for progress update and cancellation for Solr index upgrade

Core/src/org/sleuthkit/autopsy/framework/AutopsyService.java

@@ -49,9 +49,7 @@ public interface AutopsyService {
      * @param context The case context which includes things such as the case, a
      *                progress indicator for the operation, a cancellation
      *                request flag, etc.
-     *
+     * @throws org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
-     * @throws
-     * org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService.AutopsyServiceException
      */
     default void openCaseResources(CaseContext context) throws AutopsyServiceException {
         /*
@@ -65,9 +63,7 @@ public interface AutopsyService {
      * @param context The case context which includes things such as the case, a
      *                progress indicator for the operation, a cancellation
      *                request flag, etc.
-     *
+     * @throws org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
-     * @throws
-     * org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService.AutopsyServiceException
      */
     default void closeCaseResources(CaseContext context) throws AutopsyServiceException {
         /*

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java

@@ -30,11 +30,14 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.math.NumberUtils;
+import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.framework.AutopsyService;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.UNCPathUtilities;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
+import org.sleuthkit.autopsy.framework.ProgressIndicator;
+import static org.sleuthkit.autopsy.keywordsearch.SolrSearchService.checkCancellation;
 
 /**
  * This class handles the task of finding and identifying KWS index folders.
@@ -108,11 +111,37 @@ class IndexFinder {
         return bestCandidateIndex;
     }
 
-    String copyIndexAndConfigSet(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException {
+    /**
+     * Creates a copy of an existing Solr index as well as a reference copy of Solr config set.
+     * 
+     * @param indexToUpgrade Index object to create a copy of
+     * @param context AutopsyService.CaseContext object
+     * @param numCompletedWorkUnits Number of completed progress units so far
+     *
+     * @return
+     *
+     * @throws
+     * org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
+     */
+    @NbBundle.Messages({
+        "SolrSearch.copyIndex.msg=Copying existing text index",
+        "SolrSearch.copyConfigSet.msg=Copying Solr config set",})
+    String copyIndexAndConfigSet(Index indexToUpgrade, AutopsyService.CaseContext context, int numCompletedWorkUnits) throws AutopsyService.AutopsyServiceException {
+        
+        ProgressIndicator progress = context.getProgressIndicator();
+        
+        // Check for cancellation at whatever points are feasible
+        checkCancellation(context);
+        
         // Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index
-        String newIndexDir = copyExistingIndex(theCase, indexToUpgrade);
+        progress.progress(Bundle.SolrSearch_copyIndex_msg(), numCompletedWorkUnits++);
+        String newIndexDir = copyExistingIndex(context.getCase(), indexToUpgrade);
+
+        // Check for cancellation at whatever points are feasible
+        checkCancellation(context);
 
         // Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset
+        progress.progress(Bundle.SolrSearch_copyConfigSet_msg(), numCompletedWorkUnits++);
         createReferenceConfigSetCopy(new File(newIndexDir).getParent());
         
         return newIndexDir;
@@ -163,7 +192,6 @@ class IndexFinder {
                     if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) {
                         logger.log(Level.WARNING, "Unable to locate KWS config set in order to create a reference copy"); //NON-NLS
                         return;
-                        // ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("Unable to locate the config set");
                     }
                 }
             }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java

@@ -26,10 +26,13 @@ import java.util.List;
 import java.util.logging.Level;
 import org.apache.commons.lang.math.NumberUtils;
 import org.openide.modules.InstalledFileLocator;
+import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.framework.AutopsyService;
 import org.sleuthkit.autopsy.coreutils.ExecUtil;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
+import org.sleuthkit.autopsy.framework.ProgressIndicator;
+import static org.sleuthkit.autopsy.keywordsearch.SolrSearchService.checkCancellation;
 
 /**
  * This class handles the task of upgrading old indexes to the latest supported
@@ -44,20 +47,52 @@ class IndexUpgrader {
         JAVA_PATH = PlatformUtil.getJavaPath();
     }
 
-    Index performIndexUpgrade(String newIndexDir, Index indexToUpgrade, String tempResultsDir) throws AutopsyService.AutopsyServiceException {
+    /**
-        // ELTODO Check for cancellation at whatever points are feasible
+     * Perform Solr text index upgrade to the latest supported version of Solr.
+     *
+     * @param newIndexDir Full path to directory of Solr index to be upgraded
+     * @param indexToUpgrade Index object of the existing Solr index
+     * @param context AutopsyService.CaseContext object
+     * @param numCompletedWorkUnits Number of completed progress units so far
+     *
+     * @return Index object of the upgraded index
+     *
+     * @throws
+     * org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
+     */
+    @NbBundle.Messages({
+        "SolrSearch.upgrade4to5.msg=Upgrading existing text index from Solr 4 to Solr 5",
+        "SolrSearch.upgrade5to6.msg=Upgrading existing text index from Solr 5 to Solr 6",
+        "SolrSearch.upgradeFailed.msg=Upgrade of existing Solr text index failed, deleting temporary directories",})
+    Index performIndexUpgrade(String newIndexDir, Index indexToUpgrade, AutopsyService.CaseContext context, int numCompletedWorkUnits) throws AutopsyService.AutopsyServiceException {
+
+        ProgressIndicator progress = context.getProgressIndicator();
         
         // Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solrX_schema_Y/index
+        String tempResultsDir = context.getCase().getTempDirectory();
         File tmpDir = Paths.get(tempResultsDir, "IndexUpgrade").toFile(); //NON-NLS
         tmpDir.mkdirs();
 
         Index upgradedIndex;
         double currentSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
         try {
+            
+            // Check for cancellation at whatever points are feasible
+            checkCancellation(context);
+            
+            // create process terminator that will monitor the cancellation flag
+            UserCancelledProcessTerminator terminatior = new UserCancelledProcessTerminator(context);
+            
             // upgrade from Solr 4 to 5
-            currentSolrVersion = upgradeSolrIndexVersion4to5(currentSolrVersion, newIndexDir, tempResultsDir);
+            progress.progress(Bundle.SolrSearch_upgrade4to5_msg(), numCompletedWorkUnits++);
+            currentSolrVersion = upgradeSolrIndexVersion4to5(currentSolrVersion, newIndexDir, tempResultsDir, terminatior);
+
+            // Check for cancellation at whatever points are feasible
+            checkCancellation(context);
+            
             // upgrade from Solr 5 to 6
-            currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir);
+            progress.progress(Bundle.SolrSearch_upgrade5to6_msg(), numCompletedWorkUnits++);
+            currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir, terminatior);
             
             // create upgraded index object
             upgradedIndex = new Index(newIndexDir, Double.toString(currentSolrVersion), indexToUpgrade.getSchemaVersion());
@@ -68,6 +103,7 @@ class IndexUpgrader {
         } finally {
             if (currentSolrVersion != NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) {
                 // upgrade did not complete, delete the new index directories
+                progress.progress(Bundle.SolrSearch_upgradeFailed_msg(), numCompletedWorkUnits);
                 upgradedIndex = null;
                 if (!new File(newIndexDir).delete()) {
                     logger.log(Level.SEVERE, "Unable to delete folder {0}", newIndexDir); //NON-NLS
@@ -83,10 +119,11 @@ class IndexUpgrader {
      * @param currentIndexVersion Current Solr index version
      * @param solr4IndexPath Full path to Solr v4 index directory
      * @param tempResultsDir Path to directory where to store log output
+     * @param terminatior Implementation of ExecUtil.ProcessTerminator to terminate upgrade process
      *
      * @return The new Solr index version.
      */
-    private double upgradeSolrIndexVersion4to5(double currentIndexVersion, String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
+    private double upgradeSolrIndexVersion4to5(double currentIndexVersion, String solr4IndexPath, String tempResultsDir, UserCancelledProcessTerminator terminatior) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
 
         if (currentIndexVersion != 4.0) {
             return currentIndexVersion;
@@ -119,7 +156,7 @@ class IndexUpgrader {
         ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
         processBuilder.redirectOutput(new File(outputFileFullPath));
         processBuilder.redirectError(new File(errFileFullPath));
-        ExecUtil.execute(processBuilder);
+        ExecUtil.execute(processBuilder, terminatior);
 
         // alternatively can execute lucene upgrade command from the folder where lucene jars are located
         // java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
@@ -132,10 +169,11 @@ class IndexUpgrader {
      * @param currentIndexVersion Current Solr index version
      * @param solr5IndexPath Full path to Solr v5 index directory
      * @param tempResultsDir Path to directory where to store log output
+     * @param terminatior Implementation of ExecUtil.ProcessTerminator to terminate upgrade process
      *
      * @return The new Solr index version.
      */
-    private double upgradeSolrIndexVersion5to6(double currentIndexVersion, String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
+    private double upgradeSolrIndexVersion5to6(double currentIndexVersion, String solr5IndexPath, String tempResultsDir, UserCancelledProcessTerminator terminatior) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
         if (currentIndexVersion != 5.0) {
             return currentIndexVersion;
         }
@@ -167,11 +205,30 @@ class IndexUpgrader {
         ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
         processBuilder.redirectOutput(new File(outputFileFullPath));
         processBuilder.redirectError(new File(errFileFullPath));
-        ExecUtil.execute(processBuilder);
+        ExecUtil.execute(processBuilder, terminatior);
 
         // alternatively can execute lucene upgrade command from the folder where lucene jars are located
         // java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
         return 6.0;
     }
     
+    /**
+     * Process terminator that can be used to kill Solr index upgrade processes 
+     * if a user has requested to cancel the upgrade.
+     */
+    private class UserCancelledProcessTerminator implements ExecUtil.ProcessTerminator {
+
+        AutopsyService.CaseContext context = null;
+        UserCancelledProcessTerminator(AutopsyService.CaseContext context) {
+            this.context = context;
+        }
+        
+        @Override
+        public boolean shouldTerminateProcess() {
+            if (context.cancelRequested()) {
+                return true;
+            }
+            return false;
+        }        
+    }
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java

@@ -31,6 +31,7 @@ import org.openide.util.lookup.ServiceProviders;
 import org.sleuthkit.autopsy.core.RuntimeProperties;
 import org.sleuthkit.autopsy.framework.AutopsyService;
 import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.autopsy.framework.ProgressIndicator;
 import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
 import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
 import org.sleuthkit.datamodel.BlackboardArtifact;
@@ -149,29 +150,62 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
     public void close() throws IOException {
     }
 
+    /**
+     * Checks whether user has requested to cancel Solr core open/create/upgrade
+     * process. Throws exception if cancellation was requested.
+     *
+     * @param context CaseContext object
+     *
+     * @throws
+     * org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
+     */
+    static void checkCancellation(CaseContext context) throws AutopsyServiceException {
+        if (context.cancelRequested()) {
+            throw new AutopsyServiceException("Cancellation requested by user");
+        }
+    }
+
     /**
      *
      * @param context
-     * @throws org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
+     *
+     * @throws
+     * org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
      */
     @Override
+    @NbBundle.Messages({
+        "SolrSearch.findingIndexes.msg=Looking for existing text index directories",
+        "SolrSearch.creatingNewIndex.msg=Creating new text index",
+        "SolrSearch.indentifyingIndex.msg=Identifying text index for upgrade",
+        "SolrSearch.openCore.msg=Creating/Opening text index",
+        "SolrSearch.complete.msg=Text index successfully opened"})
     public void openCaseResources(CaseContext context) throws AutopsyServiceException {
         /*
          * Autopsy service providers may not have case-level resources.
          */
 
+        ProgressIndicator progress = context.getProgressIndicator();
+        int totalNumProgressUnits = 8;
+        int progressUnitsCompleted = 1;
+
         // do a case subdirectory search to check for the existence and upgrade status of KWS indexes
+        progress.start(Bundle.SolrSearch_findingIndexes_msg(), totalNumProgressUnits);
         IndexFinder indexFinder = new IndexFinder();
         List<Index> indexes = indexFinder.findAllIndexDirs(context.getCase());
 
+        // Check for cancellation at whatever points are feasible
+        checkCancellation(context);
+
         // check if index needs upgrade
         Index currentVersionIndex;
         if (indexes.isEmpty()) {
             // new case that doesn't have an existing index. create new index folder
+            progress.progress(Bundle.SolrSearch_creatingNewIndex_msg(), progressUnitsCompleted++);
             currentVersionIndex = IndexFinder.createLatestVersionIndexDir(context.getCase());
             currentVersionIndex.setNewIndex(true);
         } else {
             // check if one of the existing indexes is for latest Solr version and schema
+            progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted++);
             currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes);
             if (currentVersionIndex == null) {
                 // found existing index(es) but none were for latest Solr version and schema version
@@ -181,6 +215,9 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
                     throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
                 }
 
+                // Check for cancellation at whatever points are feasible
+                checkCancellation(context);
+
                 double currentSolrVersion = NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion());
                 double indexSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
                 if (indexSolrVersion > currentSolrVersion) {
@@ -212,13 +249,13 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
                         }
                     }
 
-                    // ELTODO Check for cancellation at whatever points are feasible
                     // Copy the existing index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/
-                    String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade);
+                    String newIndexDir = indexFinder.copyIndexAndConfigSet(indexToUpgrade, context, progressUnitsCompleted);
+                    progressUnitsCompleted += 2; // add progress increments for copying existing index and config set
 
                     // upgrade the existing index to the latest supported Solr version
                     IndexUpgrader indexUpgrader = new IndexUpgrader();
-                    currentVersionIndex = indexUpgrader.performIndexUpgrade(newIndexDir, indexToUpgrade, context.getCase().getTempDirectory());
+                    currentVersionIndex = indexUpgrader.performIndexUpgrade(newIndexDir, indexToUpgrade, context, progressUnitsCompleted);
                     if (currentVersionIndex == null) {
                         throw new AutopsyServiceException("Unable to upgrade index to the latest version of Solr");
                     }
@@ -226,12 +263,18 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
             }
         }
 
+        // Check for cancellation at whatever points are feasible
+        checkCancellation(context);
+        
         // open core
         try {
+            progress.progress(Bundle.SolrSearch_openCore_msg(), totalNumProgressUnits - 1);
             KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex);
         } catch (Exception ex) {
             throw new AutopsyServiceException(String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex);
         }
+        
+        progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);
     }
 
     /**