diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 6410a269ce..cd984c2155 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1489,9 +1489,7 @@ public class Case implements SleuthkitCase.ErrorObserver { for (AutopsyService service : Lookup.getDefault().lookupAll(AutopsyService.class)) { try { serviceName = service.getServiceName(); - if (!serviceName.equals("Solr Keyword Search Service")) { - service.openCaseResources(context); - } + service.openCaseResources(context); } catch (AutopsyService.AutopsyServiceException ex) { Case.logger.log(Level.SEVERE, String.format("%s service failed to open case resources", serviceName), ex); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index d6fc2eabed..2bf824795e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -313,5 +313,7 @@ GlobalEditListPanel.keywordDupesSkipped.text={0} keyword was already in the list GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list. GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again. GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again. -SolrSearchService.IndexUpgradeDialog.title=Index Upgrade Required In Order To Open Case -SolrSearchService.IndexUpgradeDialog.msg=Index upgrade can be a very lengthy operation that involves copying existing index and calling third party tools to upgrade it.
Upon upgrade you will be able to see existing keyword search results and perform literal keyword searches on the existing index.
However, you will not be able to add new text to the index or performing regex searches.
You must create a new case and re-run Keyword Search Ingest Module if you want to index new text or performing regex searches.
Do you wish to proceed with the index upgrade? \ No newline at end of file +SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case +SolrSearchService.IndexUpgradeDialog.msg=The text index upgrade can take some time.
When completed, you will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed with the index upgrade? +SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only +SolrSearchService.IndexReadOnlyDialog.msg=The text index for this case is read-only.
You will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed? diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java new file mode 100644 index 0000000000..67cba2aaf2 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java @@ -0,0 +1,93 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +/** + * This class encapsulates KWS index data. + */ +class Index { + + private String indexPath; + private String schemaVersion; + private String solrVersion; + + Index() { + this.indexPath = ""; + this.solrVersion = ""; + this.schemaVersion = ""; + } + + Index(String indexPath, String solrVersion, String schemaVersion) { + this.indexPath = indexPath; + this.solrVersion = solrVersion; + this.schemaVersion = schemaVersion; + } + + /** + * @return the indexPath + */ + String getIndexPath() { + return indexPath; + } + + /** + * @param indexPath the indexPath to set + */ + void setIndexPath(String indexPath) { + this.indexPath = indexPath; + } + + /** + * @return the schemaVersion + */ + String getSchemaVersion() { + return schemaVersion; + } + + /** + * @param schemaVersion the schemaVersion to set + */ + void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } + + /** + * @return the solrVersion + */ + String getSolrVersion() { + return solrVersion; + } + + /** + * @param solrVersion the solrVersion to set + */ + void setSolrVersion(String solrVersion) { + this.solrVersion = solrVersion; + } + + /** + * @param true if all Index fields are set, false otherwise + */ + boolean isIndexDataPopulated() { + if (!this.indexPath.isEmpty() && !this.solrVersion.isEmpty() && !this.schemaVersion.isEmpty()) { + return true; + } + return false; + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index affa77607e..9502da5c76 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.keywordsearch; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -28,7 +29,7 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; -import org.openide.util.NbBundle; +import org.apache.commons.lang.math.NumberUtils; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.Logger; @@ -41,13 +42,13 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; class IndexFinder { private static final Logger logger = Logger.getLogger(IndexFinder.class.getName()); - private UNCPathUtilities uncPathUtilities; + private final UNCPathUtilities uncPathUtilities; private static final String KWS_OUTPUT_FOLDER_NAME = "keywordsearch"; private static final String KWS_DATA_FOLDER_NAME = "data"; private static final String INDEX_FOLDER_NAME = "index"; private static final String CURRENT_SOLR_VERSION = "6"; private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.0"; - private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr\\d{1,2}_schema_\\d{1,2}.\\d{1,2}$"); + private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr(\\d{1,2})_schema_(\\d{1,2}\\.\\d{1,2})$"); // If SOLR_HOME environment variable doesn't exist, try these relative paths to find Solr config sets: private static final String RELATIVE_PATH_TO_CONFIG_SET = "autopsy/solr/solr/configsets/"; private static final String RELATIVE_PATH_TO_CONFIG_SET_2 = "release/solr/solr/configsets/"; @@ -64,19 +65,52 @@ class IndexFinder { return CURRENT_SOLR_SCHEMA_VERSION; } - static String findLatestVersionIndexDir(List allIndexes) { + static Index findLatestVersionIndexDir(List allIndexes) { String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; - for (String path : allIndexes) { + for (Index index : allIndexes) { + String path = index.getIndexPath(); if (path.contains(indexFolderName)) { - return path; + return index; } } - return ""; + return new Index(); + } + + static Index createLatestVersionIndexDir(Case theCase) { + String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; + // new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index" + File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS + targetDirPath.mkdirs(); + return new Index(targetDirPath.getAbsolutePath(), CURRENT_SOLR_VERSION, CURRENT_SOLR_SCHEMA_VERSION); + } + + static Index identifyIndexToUpgrade(List allIndexes) { + /* NOTE: All of the following paths are valid multi-user index paths: + (Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index + */ + Index bestCandidateIndex = new Index(); + double solrVerFound = 0.0; + double schemaVerFound = 0.0; + for (Index index : allIndexes) { + // higher Solr version takes priority because it may negate index upgrade + if (NumberUtils.toDouble(index.getSolrVersion()) >= solrVerFound) { + // if same solr version, pick the one with highest schema version + if (NumberUtils.toDouble(index.getSchemaVersion()) >= schemaVerFound) { + bestCandidateIndex = index; + solrVerFound = NumberUtils.toDouble(index.getSolrVersion()); + schemaVerFound = NumberUtils.toDouble(index.getSchemaVersion()); + } + } + } + return bestCandidateIndex; } - String copyIndexAndConfigSet(Case theCase, String oldIndexDir) throws AutopsyService.AutopsyServiceException { + String copyIndexAndConfigSet(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException { // Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index - String newIndexDir = createReferenceIndexCopy(theCase, oldIndexDir); + String newIndexDir = copyExistingIndex(theCase, indexToUpgrade); // Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset createReferenceConfigSetCopy(new File(newIndexDir).getParent()); @@ -84,9 +118,9 @@ class IndexFinder { return newIndexDir; } - private String createReferenceIndexCopy(Case theCase, String indexPath) throws AutopsyService.AutopsyServiceException { - logger.log(Level.INFO, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS - String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; + private static String copyExistingIndex(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException { + // folder name for the upgraded index should be latest Solr version BUT schema verion of the existing index + String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + indexToUpgrade.getSchemaVersion(); try { // new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index" File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS @@ -95,22 +129,19 @@ class IndexFinder { List contents = getAllContentsInFolder(targetDirPath.getAbsolutePath()); if (!contents.isEmpty()) { // target directory is not empty - logger.log(Level.SEVERE, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS throw new AutopsyService.AutopsyServiceException("Directory to store the upgraded index must be empty " + targetDirPath.getAbsolutePath()); } } targetDirPath.mkdirs(); - FileUtils.copyDirectory(new File(indexPath), targetDirPath); + FileUtils.copyDirectory(new File(indexToUpgrade.getIndexPath()), targetDirPath); return targetDirPath.getAbsolutePath(); - } catch (Exception ex) { - logger.log(Level.SEVERE, "Error occurred while creating a reference copy of keyword search index {0}", ex); //NON-NLS + } catch (AutopsyService.AutopsyServiceException | IOException ex) { throw new AutopsyService.AutopsyServiceException("Error occurred while creating a copy of keyword search index", ex); } } // ELTODO This functionality is NTH: private void createReferenceConfigSetCopy(String indexPath) { - logger.log(Level.INFO, "Creating a reference copy of config set in {0} ", indexPath); //NON-NLS File pathToConfigSet = new File(""); try { // See if there is SOLR_HOME environment variable first @@ -122,7 +153,7 @@ class IndexFinder { // if there is no SOLR_HOME: // this will only work for Windows OS if (!PlatformUtil.isWindowsOS()) { - throw new AutopsyService.AutopsyServiceException("ELTODO"); + throw new AutopsyService.AutopsyServiceException("Creating a reference config set copy is currently a Windows-only feature"); } // config set should be located in "C:/some/directory/AutopsyXYZ/autopsy/solr/solr/configsets/" pathToConfigSet = Paths.get(System.getProperty("user.dir"), RELATIVE_PATH_TO_CONFIG_SET).toFile(); @@ -132,7 +163,7 @@ class IndexFinder { if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) { logger.log(Level.WARNING, "Unable to locate KWS config set in order to create a reference copy"); //NON-NLS return; - // ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("ELTODO"); + // ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("Unable to locate the config set"); } } } @@ -144,22 +175,21 @@ class IndexFinder { if (!pathToConfigSet.getAbsolutePath().isEmpty() && pathToConfigSet.exists()) { FileUtils.copyDirectory(pathToConfigSet, new File(indexPath)); } - } catch (Exception ex) { + } catch (AutopsyService.AutopsyServiceException | IOException ex) { // This feature is a NTH so don't re-throw - logger.log(Level.WARNING, "Error while copying KWS config set to {0}", indexPath); //NON-NLS } } /** - * Find index directory location for the case. This is done via subdirectory + * Find index directory location(s) for the case. This is done via subdirectory * search of all existing "ModuleOutput/node_name/keywordsearch/data/" * folders. * * @param theCase the case to get index dir for * - * @return List of absolute paths to all found index directories + * @return List of Index objects for each found index directory */ - List findAllIndexDirs(Case theCase) { + List findAllIndexDirs(Case theCase) { ArrayList candidateIndexDirs = new ArrayList<>(); // first find all existing "/ModuleOutput/keywordsearch/data/" folders if (theCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { @@ -172,9 +202,8 @@ class IndexFinder { X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index */ - // create a list of all sub-directories + // get a list of all folder's contents List contents = getAllContentsInFolder(theCase.getCaseDirectory()); - if (!contents.isEmpty()) { // decipher "ModuleOutput" directory name from module output path // (e.g. X:\Case\ingest4\ModuleOutput\) because there is no other way to get it... @@ -205,19 +234,68 @@ class IndexFinder { } // analyze possible index folders - ArrayList indexDirs = new ArrayList<>(); + ArrayList indexes = new ArrayList<>(); for (String path : candidateIndexDirs) { List validIndexPaths = containsValidIndexFolders(path); for (String validPath : validIndexPaths) { - indexDirs.add(convertPathToUNC(validPath)); - // there can be multiple index folders (e.g. current version and "old" version) so keep looking + String solrVersion = getSolrVersionFromIndexPath(validPath); + String schemaVersion = getSchemaVersionFromIndexPath(validPath); + if (!validPath.isEmpty() && !solrVersion.isEmpty() && !schemaVersion.isEmpty()) { + indexes.add(new Index(convertPathToUNC(validPath), solrVersion, schemaVersion)); + // there can be multiple index folders (e.g. current version and "old" version) so keep looking + } } } - return indexDirs; + return indexes; } + + String getSolrVersionFromIndexPath(String path) { + /* NOTE: All of the following paths are valid multi-user index paths: + (Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index + */ + File file = new File(path); + // sanity check - must be "index" folder + if (!file.getName().equals(INDEX_FOLDER_NAME)) { + // invalid index path + return ""; + } + String parentFolderName = file.getParentFile().getName(); + if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) { + // this is a Solr4 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + return "4"; + } + + // extract Solr version if name matches "solrX_schema_Y" format + return getSolrVersionFromIndexFolderName(parentFolderName); + } + + String getSchemaVersionFromIndexPath(String path) { + /* NOTE: All of the following paths are valid multi-user index paths: + (Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index + */ + File file = new File(path); + // sanity check - must be "index" folder + if (!file.getName().equals(INDEX_FOLDER_NAME)) { + // invalid index path + return ""; + } + String parentFolderName = file.getParentFile().getName(); + if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) { + // this is a Solr 4 schema 1.8 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + return "1.8"; + } + + // extract schema version if name matches "solrX_schema_Y" format + return getSchemaVersionFromIndexFolderName(parentFolderName); + } String convertPathToUNC(String indexDir) { - // ELTODO do we need to do this when searching for old index? if (uncPathUtilities == null) { return indexDir; } @@ -309,4 +387,34 @@ class IndexFinder { Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); return m.find(); } + + /** + * Gets Solr version number if index folder name matches the standard + * + * @param inputString The string to check. + * + * @return Solr version, empty string on error + */ + static String getSolrVersionFromIndexFolderName(String inputString) { + Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); + if (m.find()) { + return m.group(1); + } + return ""; + } + + /** + * Gets Solr schema version number if index folder name matches the standard + * + * @param inputString The string to check. + * + * @return Solr schema version, empty string on error + */ + static String getSchemaVersionFromIndexFolderName(String inputString) { + Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); + if (m.find()) { + return m.group(2); + } + return ""; + } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java index 28327d9215..4e8566e934 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,6 +24,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; +import org.apache.commons.lang.math.NumberUtils; import org.openide.modules.InstalledFileLocator; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.ExecUtil; @@ -43,65 +44,60 @@ class IndexUpgrader { JAVA_PATH = PlatformUtil.getJavaPath(); } - void performIndexUpgrade(String newIndexDir, String tempResultsDir) throws AutopsyService.AutopsyServiceException { + void performIndexUpgrade(Index indexToUpgrade, String tempResultsDir) throws AutopsyService.AutopsyServiceException { // ELTODO Check for cancellation at whatever points are feasible - + + String newIndexDir = indexToUpgrade.getIndexPath(); + // Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solrX_schema_Y/index File tmpDir = Paths.get(tempResultsDir, "IndexUpgrade").toFile(); //NON-NLS tmpDir.mkdirs(); - - boolean success = true; + + double currentSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion()); try { - // upgrade from Solr 4 to 5. If index is newer than Solr 4 then the upgrade script will throw exception right away. - upgradeSolrIndexVersion4to5(newIndexDir, tempResultsDir); + // upgrade from Solr 4 to 5 + currentSolrVersion = upgradeSolrIndexVersion4to5(currentSolrVersion, newIndexDir, tempResultsDir); + // upgrade from Solr 5 to 6 + currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir); } catch (Exception ex) { - // catch-all firewall for exceptions thrown by the Solr 4 to 5 upgrade tool itself - logger.log(Level.SEVERE, "Exception while running Sorl 4 to Solr 5 upgrade tool " + newIndexDir, ex); //NON-NLS - success = false; - } - - if (success) { - try { - // upgrade from Solr 5 to 6. This one must complete successfully in order to produce a valid Solr 6 index. - upgradeSolrIndexVersion5to6(newIndexDir, tempResultsDir); - } catch (Exception ex) { - // catch-all firewall for exceptions thrown by Solr 5 to 6 upgrade tool itself - logger.log(Level.SEVERE, "Exception while running Sorl 5 to Solr 6 upgrade tool " + newIndexDir, ex); //NON-NLS - success = false; + // catch-all firewall for exceptions thrown by Solr upgrade tools + throw new AutopsyService.AutopsyServiceException("Exception while running Solr index upgrade in " + newIndexDir, ex); //NON-NLS + } finally { + if (currentSolrVersion != NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) { + // upgrade did not complete, delete the new index directories + if (!new File(newIndexDir).delete()) { + logger.log(Level.SEVERE, "Unable to delete folder {0}", newIndexDir); //NON-NLS + } } } - - if (!success) { - // delete the new directories - new File(newIndexDir).delete(); - throw new AutopsyService.AutopsyServiceException("Failed to upgrade existing keyword search index"); - } } /** * Upgrades Solr index from version 4 to 5. * + * @param currentIndexVersion Current Solr index version * @param solr4IndexPath Full path to Solr v4 index directory * @param tempResultsDir Path to directory where to store log output * - * @return True is index upgraded successfully, false otherwise + * @return The new Solr index version. */ - private void upgradeSolrIndexVersion4to5(String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { + private double upgradeSolrIndexVersion4to5(double currentIndexVersion, String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { + if (currentIndexVersion != 4.0) { + return currentIndexVersion; + } String outputFileName = "output.txt"; logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 4 to Solr 5", solr4IndexPath); //NON-NLS // find the index upgrade tool final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr4to5IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS if (upgradeToolFolder == null) { - logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool"); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool"); } // full path to index upgrade jar file File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr4IndexUpgrade.jar").toFile(); if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) { - logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file"); } @@ -122,32 +118,34 @@ class IndexUpgrader { // alternatively can execute lucene upgrade command from the folder where lucene jars are located // java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index + return 5.0; } /** * Upgrades Solr index from version 5 to 6. * + * @param currentIndexVersion Current Solr index version * @param solr5IndexPath Full path to Solr v5 index directory * @param tempResultsDir Path to directory where to store log output * - * @return True is index upgraded successfully, false otherwise + * @return The new Solr index version. */ - private void upgradeSolrIndexVersion5to6(String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { - + private double upgradeSolrIndexVersion5to6(double currentIndexVersion, String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { + if (currentIndexVersion != 5.0) { + return currentIndexVersion; + } String outputFileName = "output.txt"; logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 5 to Solr 6", solr5IndexPath); //NON-NLS // find the index upgrade tool final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr5to6IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS if (upgradeToolFolder == null) { - logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool"); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool"); } // full path to index upgrade jar file File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr5IndexUpgrade.jar").toFile(); if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) { - logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file"); } @@ -168,6 +166,7 @@ class IndexUpgrader { // alternatively can execute lucene upgrade command from the folder where lucene jars are located // java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index + return 6.0; } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java index fa4547aae7..8119015320 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java @@ -22,7 +22,6 @@ import java.util.logging.Level; import org.openide.modules.ModuleInstall; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.keywordsearch.Server.SolrServerNoPortException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; @@ -45,8 +44,6 @@ class Installer extends ModuleInstall { //Setup the default KeywordSearch configuration files KeywordSearchSettings.setDefaults(); - Case.addEventSubscriber(Case.Events.CURRENT_CASE.toString(), new KeywordSearch.CaseChangeListener()); - final Server server = KeywordSearch.getServer(); try { server.start(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java index f491d1696e..5d5c0847bb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.io.IOException; @@ -28,10 +27,8 @@ import java.util.logging.Logger; import java.util.logging.SimpleFormatter; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.keywordsearch.KeywordSearchResultFactory.BlackboardResultWriter; /** * Wrapper over KeywordSearch Solr server singleton. The class also provides @@ -109,54 +106,4 @@ public class KeywordSearch { MessageNotifyUtil.MessageType.ERROR); } } - - /** - * Listener to create/open and close Solr cores when cases are - * created/opened and closed. - */ - static class CaseChangeListener implements PropertyChangeListener { - - @Override - public void propertyChange(PropertyChangeEvent evt) { - if (evt.getPropertyName().equals(Case.Events.CURRENT_CASE.toString())) { - if (null != evt.getOldValue()) { - /* - * A case is being closed. - */ - Case closedCase = (Case) evt.getOldValue(); - try { - BlackboardResultWriter.stopAllWriters(); - /* - * TODO (AUT-2084): The following code - * KeywordSearch.CaseChangeListener gambles that any - * BlackboardResultWriters (SwingWorkers) will complete - * in less than roughly two seconds - */ - Thread.sleep(2000); - server.closeCore(); - } catch (Exception ex) { - logger.log(Level.SEVERE, String.format("Failed to close core for %s", closedCase.getCaseDirectory()), ex); //NON-NLS - if (RuntimeProperties.coreComponentsAreActive()) { - MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.closeCore.notification.msg"), ex.getMessage()); - } - } - } - - if (null != evt.getNewValue()) { - /* - * A case is being created/opened. - */ - Case openedCase = (Case) evt.getNewValue(); - try { - server.openCoreForCase(openedCase); - } catch (Exception ex) { - logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", openedCase.getCaseDirectory()), ex); //NON-NLS - if (RuntimeProperties.coreComponentsAreActive()) { - MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage()); - } - } - } - } - } - } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index cb3ebb8d21..2c8d44c5b2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -658,15 +658,15 @@ public class Server { * Creates/opens a Solr core (index) for a case. * * @param theCase The case for which the core is to be created/opened. - * + * @param index The text index that the Solr core should be using. * * @throws KeywordSearchModuleException If an error occurs while * creating/opening the core. */ - void openCoreForCase(Case theCase) throws KeywordSearchModuleException { + void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException { currentCoreLock.writeLock().lock(); try { - currentCore = openCore(theCase); + currentCore = openCore(theCase, index); serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED); } finally { currentCoreLock.writeLock().unlock(); @@ -709,32 +709,6 @@ public class Server { } } - /** - * Get index dir location for the case - * - * @param theCase the case to get index dir for - * - * @return absolute path to index dir - */ - String geCoreDataDirPath(Case theCase) { - // ELTODO this method is going to be removed - String indexDir = theCase.getModuleDirectory() + File.separator + "keywordsearch" + File.separator + "data"; //NON-NLS - if (uncPathUtilities != null) { - // if we can check for UNC paths, do so, otherwise just return the indexDir - String result = uncPathUtilities.mappedDriveToUNC(indexDir); - if (result == null) { - uncPathUtilities.rescanDrives(); - result = uncPathUtilities.mappedDriveToUNC(indexDir); - } - if (result == null) { - return indexDir; - } - return result; - } - return indexDir; - } - - /** * ** end single-case specific methods *** */ @@ -742,15 +716,14 @@ public class Server { * Creates/opens a Solr core (index) for a case. * * @param theCase The case for which the core is to be created/opened. + * @param index The text index that the Solr core should be using. * * @return An object representing the created/opened core. * * @throws KeywordSearchModuleException If an error occurs while * creating/opening the core. */ - private Core openCore(Case theCase) throws KeywordSearchModuleException { - - // ELTODO REMOVE String indexDir = findLatestVersionIndexDir(Case.getCurrentCase()); // ELTODO + private Core openCore(Case theCase, Index index) throws KeywordSearchModuleException { try { if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) { @@ -766,9 +739,8 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex); } - String dataDir = geCoreDataDirPath(theCase); String coreName = theCase.getTextIndexName(); - return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType()); + return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, index, theCase.getCaseType()); } /** @@ -1115,7 +1087,7 @@ public class Server { * Creates/opens a Solr core (index) for a case. * * @param coreName The core name. - * @param dataDir The data directory for the core. + * @param index The text index object for the core. * @param caseType The type of the case (single-user or multi-user) for * which the core is being created/opened. * @@ -1124,9 +1096,11 @@ public class Server { * @throws KeywordSearchModuleException If an error occurs while * creating/opening the core. */ - private Core openCore(String coreName, File dataDir, CaseType caseType) throws KeywordSearchModuleException { + private Core openCore(String coreName, Index index, CaseType caseType) throws KeywordSearchModuleException { try { + + File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory if (!dataDir.exists()) { dataDir.mkdirs(); } @@ -1169,8 +1143,7 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg")); } - // ELTODO set solr and schema version of the core that is being loaded. Make that available via API. - return new Core(coreName, caseType); + return new Core(coreName, caseType, index); } catch (SolrServerException | SolrException | IOException ex) { throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex); @@ -1235,14 +1208,17 @@ public class Server { private final String name; private final CaseType caseType; + + private final Index textIndex; // the server to access a core needs to be built from a URL with the // core in it, and is only good for core-specific operations private final HttpSolrClient solrCore; - private Core(String name, CaseType caseType) { + private Core(String name, CaseType caseType, Index index) { this.name = name; this.caseType = caseType; + this.textIndex = index; this.solrCore = new Builder(currentSolrServer.getBaseURL() + "/" + name).build(); //NON-NLS diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index d2f4df78d0..4f6f4525c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,13 +18,12 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.nio.file.Paths; import java.util.List; import java.util.MissingResourceException; import java.util.logging.Level; +import org.apache.commons.lang.math.NumberUtils; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.openide.util.NbBundle; @@ -33,6 +32,7 @@ import org.openide.util.lookup.ServiceProviders; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -158,43 +158,87 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService // do a case subdirectory search to check for the existence and upgrade status of KWS indexes IndexFinder indexFinder = new IndexFinder(); - List indexDirs = indexFinder.findAllIndexDirs(context.getCase()); + List indexes = indexFinder.findAllIndexDirs(context.getCase()); // check if index needs upgrade - String currentVersionIndexDir = IndexFinder.findLatestVersionIndexDir(indexDirs); - if (currentVersionIndexDir.isEmpty()) { - - // ELTODO not sure what to do when there are multiple old indexes. grab the first one? - String oldIndexDir = indexDirs.get(0); + Index currentVersionIndex; + if (indexes.isEmpty()) { + // new case that doesn't have an existing index. create new index folder + currentVersionIndex = IndexFinder.createLatestVersionIndexDir(context.getCase()); + } else { + // check if one of the existing indexes is for latest Solr version and schema + currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes); - if (RuntimeProperties.coreComponentsAreActive()) { - //pop up a message box to indicate the restrictions on adding additional - //text and performing regex searches and give the user the option to decline the upgrade - if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"), - NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"), - KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { - // upgrade declined - throw exception - throw new AutopsyServiceException("Index upgrade was declined by user"); + if (!currentVersionIndex.isIndexDataPopulated()) { + // found existing index(es) but none were for latest Solr version and schema version + Index indexToUpgrade = IndexFinder.identifyIndexToUpgrade(indexes); + if (!indexToUpgrade.isIndexDataPopulated()) { + // unable to find index that can be upgraded + throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr"); + } + + double currentSolrVersion = NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion()); + double indexSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion()); + if (indexSolrVersion > currentSolrVersion) { + // oops! + throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr"); + } + else if (indexSolrVersion == currentSolrVersion) { + // latest Solr version but not latest schema. index should be used in read-only mode and not be upgraded. + if (RuntimeProperties.coreComponentsAreActive()) { + // pop up a message box to indicate the read-only restrictions. + if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.title"), + NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.msg"), + KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { + // case open declined - throw exception + throw new AutopsyServiceException("Case open declined by user"); + } + } + // proceed with case open + currentVersionIndex = indexToUpgrade; + } + else { + // index needs to be upgraded to latest supported version of Solr + if (RuntimeProperties.coreComponentsAreActive()) { + //pop up a message box to indicate the restrictions on adding additional + //text and performing regex searches and give the user the option to decline the upgrade + if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"), + NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"), + KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { + // upgrade declined - throw exception + throw new AutopsyServiceException("Index upgrade was declined by user"); + } + } + + // ELTODO Check for cancellation at whatever points are feasible + + // Copy the existing index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ + String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade); + + // upgrade the existing index to the latest supported Solr version + IndexUpgrader indexUpgrader = new IndexUpgrader(); + indexUpgrader.performIndexUpgrade(indexToUpgrade, context.getCase().getTempDirectory()); + + // set the upgraded index as the index to be used for this case + currentVersionIndex.setIndexPath(newIndexDir); + currentVersionIndex.setSolrVersion(IndexFinder.getCurrentSolrVersion()); + currentVersionIndex.setSchemaVersion(indexToUpgrade.getSchemaVersion()); } } - - // ELTODO Check for cancellation at whatever points are feasible - - // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ - String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), oldIndexDir); - - // upgrade the "old" index to the latest supported Solr version - IndexUpgrader indexUpgrader = new IndexUpgrader(); - indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory()); - - // set the upgraded reference index as the index to be used for this case - currentVersionIndexDir = newIndexDir; } - - // open currentVersionIndexDir index - - // execute a test query - // if failed, close the upgraded index? + + // open core + try { + KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex); + } catch (Exception ex) { + logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS + if (RuntimeProperties.coreComponentsAreActive()) { + MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage()); + } + } + + // ELTODO execute a test query + // ELTODO if failed, close the upgraded index? } /** @@ -209,6 +253,22 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService /* * Autopsy service providers may not have case-level resources. */ + try { + KeywordSearchResultFactory.BlackboardResultWriter.stopAllWriters(); + /* + * TODO (AUT-2084): The following code + * KeywordSearch.CaseChangeListener gambles that any + * BlackboardResultWriters (SwingWorkers) will complete + * in less than roughly two seconds + */ + Thread.sleep(2000); + KeywordSearch.getServer().closeCore(); + } catch (Exception ex) { + logger.log(Level.SEVERE, String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS + if (RuntimeProperties.coreComponentsAreActive()) { + MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.closeCore.notification.msg"), ex.getMessage()); + } + } } @Override