mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 18:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2490 from eugene7646/progress_and_cancellation

Browse Source 
Support for progress update and cancellation for Solr index upgrade
This commit is contained in:
Richard Cordovano 2017-01-27 15:33:48 -05:00 committed by GitHub
commit e300dcd463
4 changed files with 146 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Core/src/org/sleuthkit/autopsy/framework/AutopsyService.java
View File

@ -49,9 +49,7 @@ public interface AutopsyService {
* @param context The case context which includes things such as the case, a
* progress indicator for the operation, a cancellation
* request flag, etc.
*
* @throws
* org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService.AutopsyServiceException
* @throws org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
*/
default void openCaseResources(CaseContext context) throws AutopsyServiceException {
/*
@ -65,9 +63,7 @@ public interface AutopsyService {
* @param context The case context which includes things such as the case, a
* progress indicator for the operation, a cancellation
* request flag, etc.
*
* @throws
* org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService.AutopsyServiceException
* @throws org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
*/
default void closeCaseResources(CaseContext context) throws AutopsyServiceException {
/*

34
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java
View File

@ -30,11 +30,14 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.framework.AutopsyService;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.UNCPathUtilities;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.framework.ProgressIndicator;
import static org.sleuthkit.autopsy.keywordsearch.SolrSearchService.checkCancellation;
/**
* This class handles the task of finding and identifying KWS index folders.
@ -108,11 +111,37 @@ class IndexFinder {
return bestCandidateIndex;
}
String copyIndexAndConfigSet(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException {
/**
* Creates a copy of an existing Solr index as well as a reference copy of Solr config set.
*
* @param indexToUpgrade Index object to create a copy of
* @param context AutopsyService.CaseContext object
* @param numCompletedWorkUnits Number of completed progress units so far
*
* @return
*
* @throws
* org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
*/
@NbBundle.Messages({
"SolrSearch.copyIndex.msg=Copying existing text index",
"SolrSearch.copyConfigSet.msg=Copying Solr config set",})
String copyIndexAndConfigSet(Index indexToUpgrade, AutopsyService.CaseContext context, int numCompletedWorkUnits) throws AutopsyService.AutopsyServiceException {
ProgressIndicator progress = context.getProgressIndicator();
// Check for cancellation at whatever points are feasible
checkCancellation(context);
// Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index
String newIndexDir = copyExistingIndex(theCase, indexToUpgrade);
progress.progress(Bundle.SolrSearch_copyIndex_msg(), numCompletedWorkUnits++);
String newIndexDir = copyExistingIndex(context.getCase(), indexToUpgrade);
// Check for cancellation at whatever points are feasible
checkCancellation(context);
// Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset
progress.progress(Bundle.SolrSearch_copyConfigSet_msg(), numCompletedWorkUnits++);
createReferenceConfigSetCopy(new File(newIndexDir).getParent());
return newIndexDir;
@ -163,7 +192,6 @@ class IndexFinder {
if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) {
logger.log(Level.WARNING, "Unable to locate KWS config set in order to create a reference copy"); //NON-NLS
return;
// ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("Unable to locate the config set");
}
}
}

75
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java
View File

@ -26,10 +26,13 @@ import java.util.List;
import java.util.logging.Level;
import org.apache.commons.lang.math.NumberUtils;
import org.openide.modules.InstalledFileLocator;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.framework.AutopsyService;
import org.sleuthkit.autopsy.coreutils.ExecUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.framework.ProgressIndicator;
import static org.sleuthkit.autopsy.keywordsearch.SolrSearchService.checkCancellation;
/**
* This class handles the task of upgrading old indexes to the latest supported
@ -43,21 +46,53 @@ class IndexUpgrader {
IndexUpgrader() {
JAVA_PATH = PlatformUtil.getJavaPath();
}
Index performIndexUpgrade(String newIndexDir, Index indexToUpgrade, String tempResultsDir) throws AutopsyService.AutopsyServiceException {
// ELTODO Check for cancellation at whatever points are feasible
/**
* Perform Solr text index upgrade to the latest supported version of Solr.
*
* @param newIndexDir Full path to directory of Solr index to be upgraded
* @param indexToUpgrade Index object of the existing Solr index
* @param context AutopsyService.CaseContext object
* @param numCompletedWorkUnits Number of completed progress units so far
*
* @return Index object of the upgraded index
*
* @throws
* org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
*/
@NbBundle.Messages({
"SolrSearch.upgrade4to5.msg=Upgrading existing text index from Solr 4 to Solr 5",
"SolrSearch.upgrade5to6.msg=Upgrading existing text index from Solr 5 to Solr 6",
"SolrSearch.upgradeFailed.msg=Upgrade of existing Solr text index failed, deleting temporary directories",})
Index performIndexUpgrade(String newIndexDir, Index indexToUpgrade, AutopsyService.CaseContext context, int numCompletedWorkUnits) throws AutopsyService.AutopsyServiceException {
ProgressIndicator progress = context.getProgressIndicator();
// Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solrX_schema_Y/index
String tempResultsDir = context.getCase().getTempDirectory();
File tmpDir = Paths.get(tempResultsDir, "IndexUpgrade").toFile(); //NON-NLS
tmpDir.mkdirs();
Index upgradedIndex;
double currentSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
try {
// Check for cancellation at whatever points are feasible
checkCancellation(context);
// create process terminator that will monitor the cancellation flag
UserCancelledProcessTerminator terminatior = new UserCancelledProcessTerminator(context);
// upgrade from Solr 4 to 5
currentSolrVersion = upgradeSolrIndexVersion4to5(currentSolrVersion, newIndexDir, tempResultsDir);
progress.progress(Bundle.SolrSearch_upgrade4to5_msg(), numCompletedWorkUnits++);
currentSolrVersion = upgradeSolrIndexVersion4to5(currentSolrVersion, newIndexDir, tempResultsDir, terminatior);
// Check for cancellation at whatever points are feasible
checkCancellation(context);
// upgrade from Solr 5 to 6
currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir);
progress.progress(Bundle.SolrSearch_upgrade5to6_msg(), numCompletedWorkUnits++);
currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir, terminatior);
// create upgraded index object
upgradedIndex = new Index(newIndexDir, Double.toString(currentSolrVersion), indexToUpgrade.getSchemaVersion());
@ -68,6 +103,7 @@ class IndexUpgrader {
} finally {
if (currentSolrVersion != NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) {
// upgrade did not complete, delete the new index directories
progress.progress(Bundle.SolrSearch_upgradeFailed_msg(), numCompletedWorkUnits);
upgradedIndex = null;
if (!new File(newIndexDir).delete()) {
logger.log(Level.SEVERE, "Unable to delete folder {0}", newIndexDir); //NON-NLS
@ -83,10 +119,11 @@ class IndexUpgrader {
* @param currentIndexVersion Current Solr index version
* @param solr4IndexPath Full path to Solr v4 index directory
* @param tempResultsDir Path to directory where to store log output
* @param terminatior Implementation of ExecUtil.ProcessTerminator to terminate upgrade process
*
* @return The new Solr index version.
*/
private double upgradeSolrIndexVersion4to5(double currentIndexVersion, String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
private double upgradeSolrIndexVersion4to5(double currentIndexVersion, String solr4IndexPath, String tempResultsDir, UserCancelledProcessTerminator terminatior) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
if (currentIndexVersion != 4.0) {
return currentIndexVersion;
@ -119,7 +156,7 @@ class IndexUpgrader {
ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
processBuilder.redirectOutput(new File(outputFileFullPath));
processBuilder.redirectError(new File(errFileFullPath));
ExecUtil.execute(processBuilder);
ExecUtil.execute(processBuilder, terminatior);
// alternatively can execute lucene upgrade command from the folder where lucene jars are located
// java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
@ -132,10 +169,11 @@ class IndexUpgrader {
* @param currentIndexVersion Current Solr index version
* @param solr5IndexPath Full path to Solr v5 index directory
* @param tempResultsDir Path to directory where to store log output
* @param terminatior Implementation of ExecUtil.ProcessTerminator to terminate upgrade process
*
* @return The new Solr index version.
*/
private double upgradeSolrIndexVersion5to6(double currentIndexVersion, String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
private double upgradeSolrIndexVersion5to6(double currentIndexVersion, String solr5IndexPath, String tempResultsDir, UserCancelledProcessTerminator terminatior) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
if (currentIndexVersion != 5.0) {
return currentIndexVersion;
}
@ -167,11 +205,30 @@ class IndexUpgrader {
ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
processBuilder.redirectOutput(new File(outputFileFullPath));
processBuilder.redirectError(new File(errFileFullPath));
ExecUtil.execute(processBuilder);
ExecUtil.execute(processBuilder, terminatior);
// alternatively can execute lucene upgrade command from the folder where lucene jars are located
// java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
return 6.0;
}
/**
* Process terminator that can be used to kill Solr index upgrade processes
* if a user has requested to cancel the upgrade.
*/
private class UserCancelledProcessTerminator implements ExecUtil.ProcessTerminator {
AutopsyService.CaseContext context = null;
UserCancelledProcessTerminator(AutopsyService.CaseContext context) {
this.context = context;
}
@Override
public boolean shouldTerminateProcess() {
if (context.cancelRequested()) {
return true;
}
return false;
}
}
}

51
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -31,6 +31,7 @@ import org.openide.util.lookup.ServiceProviders;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.framework.AutopsyService;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.framework.ProgressIndicator;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
import org.sleuthkit.datamodel.BlackboardArtifact;
@ -149,29 +150,62 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
public void close() throws IOException {
}
/**
* Checks whether user has requested to cancel Solr core open/create/upgrade
* process. Throws exception if cancellation was requested.
*
* @param context CaseContext object
*
* @throws
* org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
*/
static void checkCancellation(CaseContext context) throws AutopsyServiceException {
if (context.cancelRequested()) {
throw new AutopsyServiceException("Cancellation requested by user");
}
}
/**
*
* @param context
* @throws org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
*
* @throws
* org.sleuthkit.autopsy.framework.AutopsyService.AutopsyServiceException
*/
@Override
@NbBundle.Messages({
"SolrSearch.findingIndexes.msg=Looking for existing text index directories",
"SolrSearch.creatingNewIndex.msg=Creating new text index",
"SolrSearch.indentifyingIndex.msg=Identifying text index for upgrade",
"SolrSearch.openCore.msg=Creating/Opening text index",
"SolrSearch.complete.msg=Text index successfully opened"})
public void openCaseResources(CaseContext context) throws AutopsyServiceException {
/*
* Autopsy service providers may not have case-level resources.
*/
ProgressIndicator progress = context.getProgressIndicator();
int totalNumProgressUnits = 8;
int progressUnitsCompleted = 1;
// do a case subdirectory search to check for the existence and upgrade status of KWS indexes
progress.start(Bundle.SolrSearch_findingIndexes_msg(), totalNumProgressUnits);
IndexFinder indexFinder = new IndexFinder();
List<Index> indexes = indexFinder.findAllIndexDirs(context.getCase());
// Check for cancellation at whatever points are feasible
checkCancellation(context);
// check if index needs upgrade
Index currentVersionIndex;
if (indexes.isEmpty()) {
// new case that doesn't have an existing index. create new index folder
progress.progress(Bundle.SolrSearch_creatingNewIndex_msg(), progressUnitsCompleted++);
currentVersionIndex = IndexFinder.createLatestVersionIndexDir(context.getCase());
currentVersionIndex.setNewIndex(true);
} else {
// check if one of the existing indexes is for latest Solr version and schema
progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted++);
currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes);
if (currentVersionIndex == null) {
// found existing index(es) but none were for latest Solr version and schema version
@ -181,6 +215,9 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
}
// Check for cancellation at whatever points are feasible
checkCancellation(context);
double currentSolrVersion = NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion());
double indexSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
if (indexSolrVersion > currentSolrVersion) {
@ -212,13 +249,13 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
}
}
// ELTODO Check for cancellation at whatever points are feasible
// Copy the existing index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/
String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade);
String newIndexDir = indexFinder.copyIndexAndConfigSet(indexToUpgrade, context, progressUnitsCompleted);
progressUnitsCompleted += 2; // add progress increments for copying existing index and config set
// upgrade the existing index to the latest supported Solr version
IndexUpgrader indexUpgrader = new IndexUpgrader();
currentVersionIndex = indexUpgrader.performIndexUpgrade(newIndexDir, indexToUpgrade, context.getCase().getTempDirectory());
currentVersionIndex = indexUpgrader.performIndexUpgrade(newIndexDir, indexToUpgrade, context, progressUnitsCompleted);
if (currentVersionIndex == null) {
throw new AutopsyServiceException("Unable to upgrade index to the latest version of Solr");
}
@ -226,12 +263,18 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
}
}
// Check for cancellation at whatever points are feasible
checkCancellation(context);
// open core
try {
progress.progress(Bundle.SolrSearch_openCore_msg(), totalNumProgressUnits - 1);
KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex);
} catch (Exception ex) {
throw new AutopsyServiceException(String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex);
}
progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);
}
/**