mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 10:17:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2470 from eugene7646/index_upgrade_3

Browse Source 
Solr 4 to 6 index upgrade
This commit is contained in:
Richard Cordovano 2017-01-19 15:56:52 -05:00 committed by GitHub
commit 9ab0a9c75d
9 changed files with 381 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Core/src/org/sleuthkit/autopsy/casemodule/Case.java
View File

@ -1489,9 +1489,7 @@ public class Case implements SleuthkitCase.ErrorObserver {
for (AutopsyService service : Lookup.getDefault().lookupAll(AutopsyService.class)) { for (AutopsyService service : Lookup.getDefault().lookupAll(AutopsyService.class)) {
try { try {
serviceName = service.getServiceName(); serviceName = service.getServiceName();
if (!serviceName.equals("Solr Keyword Search Service")) { service.openCaseResources(context);
service.openCaseResources(context);
}
} catch (AutopsyService.AutopsyServiceException ex) { } catch (AutopsyService.AutopsyServiceException ex) {
Case.logger.log(Level.SEVERE, String.format("%s service failed to open case resources", serviceName), ex); Case.logger.log(Level.SEVERE, String.format("%s service failed to open case resources", serviceName), ex);
} }

6
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
View File

@ -313,5 +313,7 @@ GlobalEditListPanel.keywordDupesSkipped.text={0} keyword was already in the list
GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list. GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list.
GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again. GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again.
GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again. GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again.
SolrSearchService.IndexUpgradeDialog.title=Index Upgrade Required In Order To Open Case SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case
SolrSearchService.IndexUpgradeDialog.msg=<html>Index upgrade can be a very lengthy operation that involves copying existing index and calling third party tools to upgrade it. <br />Upon upgrade you will be able to see existing keyword search results and perform literal keyword searches on the existing index.<br />However, you will not be able to add new text to the index or performing regex searches.<br /> You must create a new case and re-run Keyword Search Ingest Module if you want to index new text or performing regex searches.<br /> Do you wish to proceed with the index upgrade?</html> SolrSearchService.IndexUpgradeDialog.msg=<html>The text index upgrade can take some time. <br />When completed, you will be able to see existing keyword search results and perform literal keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application. Do you wish to proceed with the index upgrade?</html>
SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only
SolrSearchService.IndexReadOnlyDialog.msg=<html>The text index for this case is read-only. <br />You will be able to see existing keyword search results and perform literal keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application. Do you wish to proceed?</html>

93
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
/**
* This class encapsulates KWS index data.
*/
class Index {
private String indexPath;
private String schemaVersion;
private String solrVersion;
Index() {
this.indexPath = "";
this.solrVersion = "";
this.schemaVersion = "";
}
Index(String indexPath, String solrVersion, String schemaVersion) {
this.indexPath = indexPath;
this.solrVersion = solrVersion;
this.schemaVersion = schemaVersion;
}
/**
* @return the indexPath
*/
String getIndexPath() {
return indexPath;
}
/**
* @param indexPath the indexPath to set
*/
void setIndexPath(String indexPath) {
this.indexPath = indexPath;
}
/**
* @return the schemaVersion
*/
String getSchemaVersion() {
return schemaVersion;
}
/**
* @param schemaVersion the schemaVersion to set
*/
void setSchemaVersion(String schemaVersion) {
this.schemaVersion = schemaVersion;
}
/**
* @return the solrVersion
*/
String getSolrVersion() {
return solrVersion;
}
/**
* @param solrVersion the solrVersion to set
*/
void setSolrVersion(String solrVersion) {
this.solrVersion = solrVersion;
}
/**
* @param true if all Index fields are set, false otherwise
*/
boolean isIndexDataPopulated() {
if (!this.indexPath.isEmpty() && !this.solrVersion.isEmpty() && !this.schemaVersion.isEmpty()) {
return true;
}
return false;
}
}

172
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java
View File

@ -1,7 +1,7 @@
/* /*
* Autopsy Forensic Browser * Autopsy Forensic Browser
* *
* Copyright 2011-2016 Basis Technology Corp. * Copyright 2011-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org * Contact: carrier <at> sleuthkit <dot> org
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.keywordsearch; package org.sleuthkit.autopsy.keywordsearch;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -28,7 +29,7 @@ import java.util.logging.Level;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.openide.util.NbBundle; import org.apache.commons.lang.math.NumberUtils;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
@ -41,13 +42,13 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil;
class IndexFinder { class IndexFinder {
private static final Logger logger = Logger.getLogger(IndexFinder.class.getName()); private static final Logger logger = Logger.getLogger(IndexFinder.class.getName());
private UNCPathUtilities uncPathUtilities; private final UNCPathUtilities uncPathUtilities;
private static final String KWS_OUTPUT_FOLDER_NAME = "keywordsearch"; private static final String KWS_OUTPUT_FOLDER_NAME = "keywordsearch";
private static final String KWS_DATA_FOLDER_NAME = "data"; private static final String KWS_DATA_FOLDER_NAME = "data";
private static final String INDEX_FOLDER_NAME = "index"; private static final String INDEX_FOLDER_NAME = "index";
private static final String CURRENT_SOLR_VERSION = "6"; private static final String CURRENT_SOLR_VERSION = "6";
private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.0"; private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.0";
private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr\\d{1,2}_schema_\\d{1,2}.\\d{1,2}$"); private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr(\\d{1,2})_schema_(\\d{1,2}\\.\\d{1,2})$");
// If SOLR_HOME environment variable doesn't exist, try these relative paths to find Solr config sets: // If SOLR_HOME environment variable doesn't exist, try these relative paths to find Solr config sets:
private static final String RELATIVE_PATH_TO_CONFIG_SET = "autopsy/solr/solr/configsets/"; private static final String RELATIVE_PATH_TO_CONFIG_SET = "autopsy/solr/solr/configsets/";
private static final String RELATIVE_PATH_TO_CONFIG_SET_2 = "release/solr/solr/configsets/"; private static final String RELATIVE_PATH_TO_CONFIG_SET_2 = "release/solr/solr/configsets/";
@ -64,19 +65,52 @@ class IndexFinder {
return CURRENT_SOLR_SCHEMA_VERSION; return CURRENT_SOLR_SCHEMA_VERSION;
} }
static String findLatestVersionIndexDir(List<String> allIndexes) { static Index findLatestVersionIndexDir(List<Index> allIndexes) {
String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION;
for (String path : allIndexes) { for (Index index : allIndexes) {
String path = index.getIndexPath();
if (path.contains(indexFolderName)) { if (path.contains(indexFolderName)) {
return path; return index;
} }
} }
return ""; return new Index();
}
static Index createLatestVersionIndexDir(Case theCase) {
String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION;
// new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index"
File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS
targetDirPath.mkdirs();
return new Index(targetDirPath.getAbsolutePath(), CURRENT_SOLR_VERSION, CURRENT_SOLR_SCHEMA_VERSION);
}
static Index identifyIndexToUpgrade(List<Index> allIndexes) {
/* NOTE: All of the following paths are valid multi-user index paths:
(Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index
*/
Index bestCandidateIndex = new Index();
double solrVerFound = 0.0;
double schemaVerFound = 0.0;
for (Index index : allIndexes) {
// higher Solr version takes priority because it may negate index upgrade
if (NumberUtils.toDouble(index.getSolrVersion()) >= solrVerFound) {
// if same solr version, pick the one with highest schema version
if (NumberUtils.toDouble(index.getSchemaVersion()) >= schemaVerFound) {
bestCandidateIndex = index;
solrVerFound = NumberUtils.toDouble(index.getSolrVersion());
schemaVerFound = NumberUtils.toDouble(index.getSchemaVersion());
}
}
}
return bestCandidateIndex;
} }
String copyIndexAndConfigSet(Case theCase, String oldIndexDir) throws AutopsyService.AutopsyServiceException { String copyIndexAndConfigSet(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException {
// Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index // Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index
String newIndexDir = createReferenceIndexCopy(theCase, oldIndexDir); String newIndexDir = copyExistingIndex(theCase, indexToUpgrade);
// Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset // Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset
createReferenceConfigSetCopy(new File(newIndexDir).getParent()); createReferenceConfigSetCopy(new File(newIndexDir).getParent());
@ -84,9 +118,9 @@ class IndexFinder {
return newIndexDir; return newIndexDir;
} }
private String createReferenceIndexCopy(Case theCase, String indexPath) throws AutopsyService.AutopsyServiceException { private static String copyExistingIndex(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException {
logger.log(Level.INFO, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS // folder name for the upgraded index should be latest Solr version BUT schema verion of the existing index
String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + indexToUpgrade.getSchemaVersion();
try { try {
// new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index" // new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index"
File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS
@ -95,22 +129,19 @@ class IndexFinder {
List<File> contents = getAllContentsInFolder(targetDirPath.getAbsolutePath()); List<File> contents = getAllContentsInFolder(targetDirPath.getAbsolutePath());
if (!contents.isEmpty()) { if (!contents.isEmpty()) {
// target directory is not empty // target directory is not empty
logger.log(Level.SEVERE, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Directory to store the upgraded index must be empty " + targetDirPath.getAbsolutePath()); throw new AutopsyService.AutopsyServiceException("Directory to store the upgraded index must be empty " + targetDirPath.getAbsolutePath());
} }
} }
targetDirPath.mkdirs(); targetDirPath.mkdirs();
FileUtils.copyDirectory(new File(indexPath), targetDirPath); FileUtils.copyDirectory(new File(indexToUpgrade.getIndexPath()), targetDirPath);
return targetDirPath.getAbsolutePath(); return targetDirPath.getAbsolutePath();
} catch (Exception ex) { } catch (AutopsyService.AutopsyServiceException | IOException ex) {
logger.log(Level.SEVERE, "Error occurred while creating a reference copy of keyword search index {0}", ex); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Error occurred while creating a copy of keyword search index", ex); throw new AutopsyService.AutopsyServiceException("Error occurred while creating a copy of keyword search index", ex);
} }
} }
// ELTODO This functionality is NTH: // ELTODO This functionality is NTH:
private void createReferenceConfigSetCopy(String indexPath) { private void createReferenceConfigSetCopy(String indexPath) {
logger.log(Level.INFO, "Creating a reference copy of config set in {0} ", indexPath); //NON-NLS
File pathToConfigSet = new File(""); File pathToConfigSet = new File("");
try { try {
// See if there is SOLR_HOME environment variable first // See if there is SOLR_HOME environment variable first
@ -122,7 +153,7 @@ class IndexFinder {
// if there is no SOLR_HOME: // if there is no SOLR_HOME:
// this will only work for Windows OS // this will only work for Windows OS
if (!PlatformUtil.isWindowsOS()) { if (!PlatformUtil.isWindowsOS()) {
throw new AutopsyService.AutopsyServiceException("ELTODO"); throw new AutopsyService.AutopsyServiceException("Creating a reference config set copy is currently a Windows-only feature");
} }
// config set should be located in "C:/some/directory/AutopsyXYZ/autopsy/solr/solr/configsets/" // config set should be located in "C:/some/directory/AutopsyXYZ/autopsy/solr/solr/configsets/"
pathToConfigSet = Paths.get(System.getProperty("user.dir"), RELATIVE_PATH_TO_CONFIG_SET).toFile(); pathToConfigSet = Paths.get(System.getProperty("user.dir"), RELATIVE_PATH_TO_CONFIG_SET).toFile();
@ -132,7 +163,7 @@ class IndexFinder {
if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) { if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) {
logger.log(Level.WARNING, "Unable to locate KWS config set in order to create a reference copy"); //NON-NLS logger.log(Level.WARNING, "Unable to locate KWS config set in order to create a reference copy"); //NON-NLS
return; return;
// ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("ELTODO"); // ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("Unable to locate the config set");
} }
} }
} }
@ -144,22 +175,21 @@ class IndexFinder {
if (!pathToConfigSet.getAbsolutePath().isEmpty() && pathToConfigSet.exists()) { if (!pathToConfigSet.getAbsolutePath().isEmpty() && pathToConfigSet.exists()) {
FileUtils.copyDirectory(pathToConfigSet, new File(indexPath)); FileUtils.copyDirectory(pathToConfigSet, new File(indexPath));
} }
} catch (Exception ex) { } catch (AutopsyService.AutopsyServiceException | IOException ex) {
// This feature is a NTH so don't re-throw // This feature is a NTH so don't re-throw
logger.log(Level.WARNING, "Error while copying KWS config set to {0}", indexPath); //NON-NLS
} }
} }
/** /**
* Find index directory location for the case. This is done via subdirectory * Find index directory location(s) for the case. This is done via subdirectory
* search of all existing "ModuleOutput/node_name/keywordsearch/data/" * search of all existing "ModuleOutput/node_name/keywordsearch/data/"
* folders. * folders.
* *
* @param theCase the case to get index dir for * @param theCase the case to get index dir for
* *
* @return List of absolute paths to all found index directories * @return List of Index objects for each found index directory
*/ */
List<String> findAllIndexDirs(Case theCase) { List<Index> findAllIndexDirs(Case theCase) {
ArrayList<String> candidateIndexDirs = new ArrayList<>(); ArrayList<String> candidateIndexDirs = new ArrayList<>();
// first find all existing "/ModuleOutput/keywordsearch/data/" folders // first find all existing "/ModuleOutput/keywordsearch/data/" folders
if (theCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { if (theCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) {
@ -172,9 +202,8 @@ class IndexFinder {
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index
*/ */
// create a list of all sub-directories // get a list of all folder's contents
List<File> contents = getAllContentsInFolder(theCase.getCaseDirectory()); List<File> contents = getAllContentsInFolder(theCase.getCaseDirectory());
if (!contents.isEmpty()) { if (!contents.isEmpty()) {
// decipher "ModuleOutput" directory name from module output path // decipher "ModuleOutput" directory name from module output path
// (e.g. X:\Case\ingest4\ModuleOutput\) because there is no other way to get it... // (e.g. X:\Case\ingest4\ModuleOutput\) because there is no other way to get it...
@ -205,19 +234,68 @@ class IndexFinder {
} }
// analyze possible index folders // analyze possible index folders
ArrayList<String> indexDirs = new ArrayList<>(); ArrayList<Index> indexes = new ArrayList<>();
for (String path : candidateIndexDirs) { for (String path : candidateIndexDirs) {
List<String> validIndexPaths = containsValidIndexFolders(path); List<String> validIndexPaths = containsValidIndexFolders(path);
for (String validPath : validIndexPaths) { for (String validPath : validIndexPaths) {
indexDirs.add(convertPathToUNC(validPath)); String solrVersion = getSolrVersionFromIndexPath(validPath);
// there can be multiple index folders (e.g. current version and "old" version) so keep looking String schemaVersion = getSchemaVersionFromIndexPath(validPath);
if (!validPath.isEmpty() && !solrVersion.isEmpty() && !schemaVersion.isEmpty()) {
indexes.add(new Index(convertPathToUNC(validPath), solrVersion, schemaVersion));
// there can be multiple index folders (e.g. current version and "old" version) so keep looking
}
} }
} }
return indexDirs; return indexes;
} }
String getSolrVersionFromIndexPath(String path) {
/* NOTE: All of the following paths are valid multi-user index paths:
(Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index
*/
File file = new File(path);
// sanity check - must be "index" folder
if (!file.getName().equals(INDEX_FOLDER_NAME)) {
// invalid index path
return "";
}
String parentFolderName = file.getParentFile().getName();
if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) {
// this is a Solr4 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index
return "4";
}
// extract Solr version if name matches "solrX_schema_Y" format
return getSolrVersionFromIndexFolderName(parentFolderName);
}
String getSchemaVersionFromIndexPath(String path) {
/* NOTE: All of the following paths are valid multi-user index paths:
(Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index
*/
File file = new File(path);
// sanity check - must be "index" folder
if (!file.getName().equals(INDEX_FOLDER_NAME)) {
// invalid index path
return "";
}
String parentFolderName = file.getParentFile().getName();
if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) {
// this is a Solr 4 schema 1.8 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index
return "1.8";
}
// extract schema version if name matches "solrX_schema_Y" format
return getSchemaVersionFromIndexFolderName(parentFolderName);
}
String convertPathToUNC(String indexDir) { String convertPathToUNC(String indexDir) {
// ELTODO do we need to do this when searching for old index?
if (uncPathUtilities == null) { if (uncPathUtilities == null) {
return indexDir; return indexDir;
} }
@ -309,4 +387,34 @@ class IndexFinder {
Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString);
return m.find(); return m.find();
} }
/**
* Gets Solr version number if index folder name matches the standard
*
* @param inputString The string to check.
*
* @return Solr version, empty string on error
*/
static String getSolrVersionFromIndexFolderName(String inputString) {
Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString);
if (m.find()) {
return m.group(1);
}
return "";
}
/**
* Gets Solr schema version number if index folder name matches the standard
*
* @param inputString The string to check.
*
* @return Solr schema version, empty string on error
*/
static String getSchemaVersionFromIndexFolderName(String inputString) {
Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString);
if (m.find()) {
return m.group(2);
}
return "";
}
} }

69
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java
View File

@ -1,7 +1,7 @@
/* /*
* Autopsy Forensic Browser * Autopsy Forensic Browser
* *
* Copyright 2011-2016 Basis Technology Corp. * Copyright 2011-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org * Contact: carrier <at> sleuthkit <dot> org
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
@ -24,6 +24,7 @@ import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.logging.Level; import java.util.logging.Level;
import org.apache.commons.lang.math.NumberUtils;
import org.openide.modules.InstalledFileLocator; import org.openide.modules.InstalledFileLocator;
import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService;
import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.autopsy.coreutils.ExecUtil;
@ -43,65 +44,60 @@ class IndexUpgrader {
JAVA_PATH = PlatformUtil.getJavaPath(); JAVA_PATH = PlatformUtil.getJavaPath();
} }
void performIndexUpgrade(String newIndexDir, String tempResultsDir) throws AutopsyService.AutopsyServiceException { void performIndexUpgrade(Index indexToUpgrade, String tempResultsDir) throws AutopsyService.AutopsyServiceException {
// ELTODO Check for cancellation at whatever points are feasible // ELTODO Check for cancellation at whatever points are feasible
String newIndexDir = indexToUpgrade.getIndexPath();
// Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solrX_schema_Y/index // Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solrX_schema_Y/index
File tmpDir = Paths.get(tempResultsDir, "IndexUpgrade").toFile(); //NON-NLS File tmpDir = Paths.get(tempResultsDir, "IndexUpgrade").toFile(); //NON-NLS
tmpDir.mkdirs(); tmpDir.mkdirs();
boolean success = true; double currentSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
try { try {
// upgrade from Solr 4 to 5. If index is newer than Solr 4 then the upgrade script will throw exception right away. // upgrade from Solr 4 to 5
upgradeSolrIndexVersion4to5(newIndexDir, tempResultsDir); currentSolrVersion = upgradeSolrIndexVersion4to5(currentSolrVersion, newIndexDir, tempResultsDir);
// upgrade from Solr 5 to 6
currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir);
} catch (Exception ex) { } catch (Exception ex) {
// catch-all firewall for exceptions thrown by the Solr 4 to 5 upgrade tool itself // catch-all firewall for exceptions thrown by Solr upgrade tools
logger.log(Level.SEVERE, "Exception while running Sorl 4 to Solr 5 upgrade tool " + newIndexDir, ex); //NON-NLS throw new AutopsyService.AutopsyServiceException("Exception while running Solr index upgrade in " + newIndexDir, ex); //NON-NLS
success = false; } finally {
} if (currentSolrVersion != NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) {
// upgrade did not complete, delete the new index directories
if (success) { if (!new File(newIndexDir).delete()) {
try { logger.log(Level.SEVERE, "Unable to delete folder {0}", newIndexDir); //NON-NLS
// upgrade from Solr 5 to 6. This one must complete successfully in order to produce a valid Solr 6 index. }
upgradeSolrIndexVersion5to6(newIndexDir, tempResultsDir);
} catch (Exception ex) {
// catch-all firewall for exceptions thrown by Solr 5 to 6 upgrade tool itself
logger.log(Level.SEVERE, "Exception while running Sorl 5 to Solr 6 upgrade tool " + newIndexDir, ex); //NON-NLS
success = false;
} }
} }
if (!success) {
// delete the new directories
new File(newIndexDir).delete();
throw new AutopsyService.AutopsyServiceException("Failed to upgrade existing keyword search index");
}
} }
/** /**
* Upgrades Solr index from version 4 to 5. * Upgrades Solr index from version 4 to 5.
* *
* @param currentIndexVersion Current Solr index version
* @param solr4IndexPath Full path to Solr v4 index directory * @param solr4IndexPath Full path to Solr v4 index directory
* @param tempResultsDir Path to directory where to store log output * @param tempResultsDir Path to directory where to store log output
* *
* @return True is index upgraded successfully, false otherwise * @return The new Solr index version.
*/ */
private void upgradeSolrIndexVersion4to5(String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { private double upgradeSolrIndexVersion4to5(double currentIndexVersion, String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
if (currentIndexVersion != 4.0) {
return currentIndexVersion;
}
String outputFileName = "output.txt"; String outputFileName = "output.txt";
logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 4 to Solr 5", solr4IndexPath); //NON-NLS logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 4 to Solr 5", solr4IndexPath); //NON-NLS
// find the index upgrade tool // find the index upgrade tool
final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr4to5IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr4to5IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS
if (upgradeToolFolder == null) { if (upgradeToolFolder == null) {
logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool"); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool"); throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool");
} }
// full path to index upgrade jar file // full path to index upgrade jar file
File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr4IndexUpgrade.jar").toFile(); File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr4IndexUpgrade.jar").toFile();
if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) { if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) {
logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file"); throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file");
} }
@ -122,32 +118,34 @@ class IndexUpgrader {
// alternatively can execute lucene upgrade command from the folder where lucene jars are located // alternatively can execute lucene upgrade command from the folder where lucene jars are located
// java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index // java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
return 5.0;
} }
/** /**
* Upgrades Solr index from version 5 to 6. * Upgrades Solr index from version 5 to 6.
* *
* @param currentIndexVersion Current Solr index version
* @param solr5IndexPath Full path to Solr v5 index directory * @param solr5IndexPath Full path to Solr v5 index directory
* @param tempResultsDir Path to directory where to store log output * @param tempResultsDir Path to directory where to store log output
* *
* @return True is index upgraded successfully, false otherwise * @return The new Solr index version.
*/ */
private void upgradeSolrIndexVersion5to6(String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { private double upgradeSolrIndexVersion5to6(double currentIndexVersion, String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
if (currentIndexVersion != 5.0) {
return currentIndexVersion;
}
String outputFileName = "output.txt"; String outputFileName = "output.txt";
logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 5 to Solr 6", solr5IndexPath); //NON-NLS logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 5 to Solr 6", solr5IndexPath); //NON-NLS
// find the index upgrade tool // find the index upgrade tool
final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr5to6IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr5to6IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS
if (upgradeToolFolder == null) { if (upgradeToolFolder == null) {
logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool"); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool"); throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool");
} }
// full path to index upgrade jar file // full path to index upgrade jar file
File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr5IndexUpgrade.jar").toFile(); File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr5IndexUpgrade.jar").toFile();
if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) { if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) {
logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file"); throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file");
} }
@ -168,6 +166,7 @@ class IndexUpgrader {
// alternatively can execute lucene upgrade command from the folder where lucene jars are located // alternatively can execute lucene upgrade command from the folder where lucene jars are located
// java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index // java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
return 6.0;
} }
} }

3
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java
View File

@ -22,7 +22,6 @@ import java.util.logging.Level;
import org.openide.modules.ModuleInstall; import org.openide.modules.ModuleInstall;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.openide.windows.WindowManager; import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.keywordsearch.Server.SolrServerNoPortException; import org.sleuthkit.autopsy.keywordsearch.Server.SolrServerNoPortException;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
@ -45,8 +44,6 @@ class Installer extends ModuleInstall {
//Setup the default KeywordSearch configuration files //Setup the default KeywordSearch configuration files
KeywordSearchSettings.setDefaults(); KeywordSearchSettings.setDefaults();
Case.addEventSubscriber(Case.Events.CURRENT_CASE.toString(), new KeywordSearch.CaseChangeListener());
final Server server = KeywordSearch.getServer(); final Server server = KeywordSearch.getServer();
try { try {
server.start(); server.start();

53
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java
View File

@ -18,7 +18,6 @@
*/ */
package org.sleuthkit.autopsy.keywordsearch; package org.sleuthkit.autopsy.keywordsearch;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener; import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport; import java.beans.PropertyChangeSupport;
import java.io.IOException; import java.io.IOException;
@ -28,10 +27,8 @@ import java.util.logging.Logger;
import java.util.logging.SimpleFormatter; import java.util.logging.SimpleFormatter;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchResultFactory.BlackboardResultWriter;
/** /**
* Wrapper over KeywordSearch Solr server singleton. The class also provides * Wrapper over KeywordSearch Solr server singleton. The class also provides
@ -109,54 +106,4 @@ public class KeywordSearch {
MessageNotifyUtil.MessageType.ERROR); MessageNotifyUtil.MessageType.ERROR);
} }
} }
/**
* Listener to create/open and close Solr cores when cases are
* created/opened and closed.
*/
static class CaseChangeListener implements PropertyChangeListener {
@Override
public void propertyChange(PropertyChangeEvent evt) {
if (evt.getPropertyName().equals(Case.Events.CURRENT_CASE.toString())) {
if (null != evt.getOldValue()) {
/*
* A case is being closed.
*/
Case closedCase = (Case) evt.getOldValue();
try {
BlackboardResultWriter.stopAllWriters();
/*
* TODO (AUT-2084): The following code
* KeywordSearch.CaseChangeListener gambles that any
* BlackboardResultWriters (SwingWorkers) will complete
* in less than roughly two seconds
*/
Thread.sleep(2000);
server.closeCore();
} catch (Exception ex) {
logger.log(Level.SEVERE, String.format("Failed to close core for %s", closedCase.getCaseDirectory()), ex); //NON-NLS
if (RuntimeProperties.coreComponentsAreActive()) {
MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.closeCore.notification.msg"), ex.getMessage());
}
}
}
if (null != evt.getNewValue()) {
/*
* A case is being created/opened.
*/
Case openedCase = (Case) evt.getNewValue();
try {
server.openCoreForCase(openedCase);
} catch (Exception ex) {
logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", openedCase.getCaseDirectory()), ex); //NON-NLS
if (RuntimeProperties.coreComponentsAreActive()) {
MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage());
}
}
}
}
}
}
} }

54
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -658,15 +658,15 @@ public class Server {
* Creates/opens a Solr core (index) for a case. * Creates/opens a Solr core (index) for a case.
* *
* @param theCase The case for which the core is to be created/opened. * @param theCase The case for which the core is to be created/opened.
* * @param index The text index that the Solr core should be using.
* *
* @throws KeywordSearchModuleException If an error occurs while * @throws KeywordSearchModuleException If an error occurs while
* creating/opening the core. * creating/opening the core.
*/ */
void openCoreForCase(Case theCase) throws KeywordSearchModuleException { void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
currentCoreLock.writeLock().lock(); currentCoreLock.writeLock().lock();
try { try {
currentCore = openCore(theCase); currentCore = openCore(theCase, index);
serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED); serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
} finally { } finally {
currentCoreLock.writeLock().unlock(); currentCoreLock.writeLock().unlock();
@ -709,32 +709,6 @@ public class Server {
} }
} }
/**
* Get index dir location for the case
*
* @param theCase the case to get index dir for
*
* @return absolute path to index dir
*/
String geCoreDataDirPath(Case theCase) {
// ELTODO this method is going to be removed
String indexDir = theCase.getModuleDirectory() + File.separator + "keywordsearch" + File.separator + "data"; //NON-NLS
if (uncPathUtilities != null) {
// if we can check for UNC paths, do so, otherwise just return the indexDir
String result = uncPathUtilities.mappedDriveToUNC(indexDir);
if (result == null) {
uncPathUtilities.rescanDrives();
result = uncPathUtilities.mappedDriveToUNC(indexDir);
}
if (result == null) {
return indexDir;
}
return result;
}
return indexDir;
}
/** /**
* ** end single-case specific methods *** * ** end single-case specific methods ***
*/ */
@ -742,15 +716,14 @@ public class Server {
* Creates/opens a Solr core (index) for a case. * Creates/opens a Solr core (index) for a case.
* *
* @param theCase The case for which the core is to be created/opened. * @param theCase The case for which the core is to be created/opened.
* @param index The text index that the Solr core should be using.
* *
* @return An object representing the created/opened core. * @return An object representing the created/opened core.
* *
* @throws KeywordSearchModuleException If an error occurs while * @throws KeywordSearchModuleException If an error occurs while
* creating/opening the core. * creating/opening the core.
*/ */
private Core openCore(Case theCase) throws KeywordSearchModuleException { private Core openCore(Case theCase, Index index) throws KeywordSearchModuleException {
// ELTODO REMOVE String indexDir = findLatestVersionIndexDir(Case.getCurrentCase()); // ELTODO
try { try {
if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) { if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
@ -766,9 +739,8 @@ public class Server {
throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex); throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex);
} }
String dataDir = geCoreDataDirPath(theCase);
String coreName = theCase.getTextIndexName(); String coreName = theCase.getTextIndexName();
return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType()); return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, index, theCase.getCaseType());
} }
/** /**
@ -1115,7 +1087,7 @@ public class Server {
* Creates/opens a Solr core (index) for a case. * Creates/opens a Solr core (index) for a case.
* *
* @param coreName The core name. * @param coreName The core name.
* @param dataDir The data directory for the core. * @param index The text index object for the core.
* @param caseType The type of the case (single-user or multi-user) for * @param caseType The type of the case (single-user or multi-user) for
* which the core is being created/opened. * which the core is being created/opened.
* *
@ -1124,9 +1096,11 @@ public class Server {
* @throws KeywordSearchModuleException If an error occurs while * @throws KeywordSearchModuleException If an error occurs while
* creating/opening the core. * creating/opening the core.
*/ */
private Core openCore(String coreName, File dataDir, CaseType caseType) throws KeywordSearchModuleException { private Core openCore(String coreName, Index index, CaseType caseType) throws KeywordSearchModuleException {
try { try {
File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
if (!dataDir.exists()) { if (!dataDir.exists()) {
dataDir.mkdirs(); dataDir.mkdirs();
} }
@ -1169,8 +1143,7 @@ public class Server {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg")); throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
} }
// ELTODO set solr and schema version of the core that is being loaded. Make that available via API. return new Core(coreName, caseType, index);
return new Core(coreName, caseType);
} catch (SolrServerException | SolrException | IOException ex) { } catch (SolrServerException | SolrException | IOException ex) {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex); throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
@ -1235,14 +1208,17 @@ public class Server {
private final String name; private final String name;
private final CaseType caseType; private final CaseType caseType;
private final Index textIndex;
// the server to access a core needs to be built from a URL with the // the server to access a core needs to be built from a URL with the
// core in it, and is only good for core-specific operations // core in it, and is only good for core-specific operations
private final HttpSolrClient solrCore; private final HttpSolrClient solrCore;
private Core(String name, CaseType caseType) { private Core(String name, CaseType caseType, Index index) {
this.name = name; this.name = name;
this.caseType = caseType; this.caseType = caseType;
this.textIndex = index;
this.solrCore = new Builder(currentSolrServer.getBaseURL() + "/" + name).build(); //NON-NLS this.solrCore = new Builder(currentSolrServer.getBaseURL() + "/" + name).build(); //NON-NLS

128
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -1,7 +1,7 @@
/* /*
* Autopsy Forensic Browser * Autopsy Forensic Browser
* *
* Copyright 2011-2016 Basis Technology Corp. * Copyright 2011-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org * Contact: carrier <at> sleuthkit <dot> org
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
@ -18,13 +18,12 @@
*/ */
package org.sleuthkit.autopsy.keywordsearch; package org.sleuthkit.autopsy.keywordsearch;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.InetAddress; import java.net.InetAddress;
import java.nio.file.Paths;
import java.util.List; import java.util.List;
import java.util.MissingResourceException; import java.util.MissingResourceException;
import java.util.logging.Level; import java.util.logging.Level;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
@ -33,6 +32,7 @@ import org.openide.util.lookup.ServiceProviders;
import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact;
@ -158,43 +158,87 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
// do a case subdirectory search to check for the existence and upgrade status of KWS indexes // do a case subdirectory search to check for the existence and upgrade status of KWS indexes
IndexFinder indexFinder = new IndexFinder(); IndexFinder indexFinder = new IndexFinder();
List<String> indexDirs = indexFinder.findAllIndexDirs(context.getCase()); List<Index> indexes = indexFinder.findAllIndexDirs(context.getCase());
// check if index needs upgrade // check if index needs upgrade
String currentVersionIndexDir = IndexFinder.findLatestVersionIndexDir(indexDirs); Index currentVersionIndex;
if (currentVersionIndexDir.isEmpty()) { if (indexes.isEmpty()) {
// new case that doesn't have an existing index. create new index folder
// ELTODO not sure what to do when there are multiple old indexes. grab the first one? currentVersionIndex = IndexFinder.createLatestVersionIndexDir(context.getCase());
String oldIndexDir = indexDirs.get(0); } else {
// check if one of the existing indexes is for latest Solr version and schema
currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes);
if (RuntimeProperties.coreComponentsAreActive()) { if (!currentVersionIndex.isIndexDataPopulated()) {
//pop up a message box to indicate the restrictions on adding additional // found existing index(es) but none were for latest Solr version and schema version
//text and performing regex searches and give the user the option to decline the upgrade Index indexToUpgrade = IndexFinder.identifyIndexToUpgrade(indexes);
if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"), if (!indexToUpgrade.isIndexDataPopulated()) {
NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"), // unable to find index that can be upgraded
KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
// upgrade declined - throw exception }
throw new AutopsyServiceException("Index upgrade was declined by user");
double currentSolrVersion = NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion());
double indexSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
if (indexSolrVersion > currentSolrVersion) {
// oops!
throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
}
else if (indexSolrVersion == currentSolrVersion) {
// latest Solr version but not latest schema. index should be used in read-only mode and not be upgraded.
if (RuntimeProperties.coreComponentsAreActive()) {
// pop up a message box to indicate the read-only restrictions.
if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.title"),
NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.msg"),
KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) {
// case open declined - throw exception
throw new AutopsyServiceException("Case open declined by user");
}
}
// proceed with case open
currentVersionIndex = indexToUpgrade;
}
else {
// index needs to be upgraded to latest supported version of Solr
if (RuntimeProperties.coreComponentsAreActive()) {
//pop up a message box to indicate the restrictions on adding additional
//text and performing regex searches and give the user the option to decline the upgrade
if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"),
NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"),
KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) {
// upgrade declined - throw exception
throw new AutopsyServiceException("Index upgrade was declined by user");
}
}
// ELTODO Check for cancellation at whatever points are feasible
// Copy the existing index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/
String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade);
// upgrade the existing index to the latest supported Solr version
IndexUpgrader indexUpgrader = new IndexUpgrader();
indexUpgrader.performIndexUpgrade(indexToUpgrade, context.getCase().getTempDirectory());
// set the upgraded index as the index to be used for this case
currentVersionIndex.setIndexPath(newIndexDir);
currentVersionIndex.setSolrVersion(IndexFinder.getCurrentSolrVersion());
currentVersionIndex.setSchemaVersion(indexToUpgrade.getSchemaVersion());
} }
} }
// ELTODO Check for cancellation at whatever points are feasible
// Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/
String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), oldIndexDir);
// upgrade the "old" index to the latest supported Solr version
IndexUpgrader indexUpgrader = new IndexUpgrader();
indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory());
// set the upgraded reference index as the index to be used for this case
currentVersionIndexDir = newIndexDir;
} }
// open currentVersionIndexDir index // open core
try {
// execute a test query KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex);
// if failed, close the upgraded index? } catch (Exception ex) {
logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS
if (RuntimeProperties.coreComponentsAreActive()) {
MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage());
}
}
// ELTODO execute a test query
// ELTODO if failed, close the upgraded index?
} }
/** /**
@ -209,6 +253,22 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
/* /*
* Autopsy service providers may not have case-level resources. * Autopsy service providers may not have case-level resources.
*/ */
try {
KeywordSearchResultFactory.BlackboardResultWriter.stopAllWriters();
/*
* TODO (AUT-2084): The following code
* KeywordSearch.CaseChangeListener gambles that any
* BlackboardResultWriters (SwingWorkers) will complete
* in less than roughly two seconds
*/
Thread.sleep(2000);
KeywordSearch.getServer().closeCore();
} catch (Exception ex) {
logger.log(Level.SEVERE, String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS
if (RuntimeProperties.coreComponentsAreActive()) {
MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.closeCore.notification.msg"), ex.getMessage());
}
}
} }
@Override @Override