From 247b3c96aaee881314d837a3bbd8081bfd9e22fc Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 17 Jan 2017 16:30:10 -0500 Subject: [PATCH 01/12] More work --- .../autopsy/keywordsearch/IndexFinder.java | 9 +- .../autopsy/keywordsearch/Installer.java | 2 - .../keywordsearch/SolrSearchService.java | 96 +++++++++++++------ 3 files changed, 76 insertions(+), 31 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index 72e5ddd653..dd88b79afc 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -28,7 +28,6 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; -import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.Logger; @@ -73,6 +72,14 @@ class IndexFinder { } return ""; } + + static String createLatestVersionIndexDir(Case theCase) { + String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; + // new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index" + File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS + targetDirPath.mkdirs(); + return targetDirPath.getAbsolutePath(); + } String copyIndexAndConfigSet(Case theCase, String oldIndexDir) throws AutopsyService.AutopsyServiceException { // Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java index fa4547aae7..7127faf2d5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java @@ -45,8 +45,6 @@ class Installer extends ModuleInstall { //Setup the default KeywordSearch configuration files KeywordSearchSettings.setDefaults(); - Case.addEventSubscriber(Case.Events.CURRENT_CASE.toString(), new KeywordSearch.CaseChangeListener()); - final Server server = KeywordSearch.getServer(); try { server.start(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index d2f4df78d0..62d0f93932 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -30,9 +30,11 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -161,40 +163,62 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService List indexDirs = indexFinder.findAllIndexDirs(context.getCase()); // check if index needs upgrade + String schemaVer; String currentVersionIndexDir = IndexFinder.findLatestVersionIndexDir(indexDirs); if (currentVersionIndexDir.isEmpty()) { - - // ELTODO not sure what to do when there are multiple old indexes. grab the first one? - String oldIndexDir = indexDirs.get(0); + if (indexDirs.isEmpty()) { + // new case that doesn't have an existing index. create new index folder + currentVersionIndexDir = IndexFinder.createLatestVersionIndexDir(context.getCase()); + schemaVer = IndexFinder.getCurrentSchemaVersion(); + } else { + // ELTODO not sure what to do when there are multiple old indexes. grab the first one? + // ELTODO need to handle here a case where it is Solr 6 index but not latest schema + // ELTODO figure out the schema version of the "old" index + String oldIndexDir = indexDirs.get(0); - if (RuntimeProperties.coreComponentsAreActive()) { - //pop up a message box to indicate the restrictions on adding additional - //text and performing regex searches and give the user the option to decline the upgrade - if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"), - NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"), - KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { - // upgrade declined - throw exception - throw new AutopsyServiceException("Index upgrade was declined by user"); + if (RuntimeProperties.coreComponentsAreActive()) { + //pop up a message box to indicate the restrictions on adding additional + //text and performing regex searches and give the user the option to decline the upgrade + if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"), + NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"), + KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { + // upgrade declined - throw exception + throw new AutopsyServiceException("Index upgrade was declined by user"); + } } + + // ELTODO Check for cancellation at whatever points are feasible + + // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ + String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), oldIndexDir); + + // upgrade the "old" index to the latest supported Solr version + IndexUpgrader indexUpgrader = new IndexUpgrader(); + indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory()); + + // set the upgraded reference index as the index to be used for this case + currentVersionIndexDir = newIndexDir; + // schemaVer = ; // ELTODO } - - // ELTODO Check for cancellation at whatever points are feasible - - // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ - String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), oldIndexDir); - - // upgrade the "old" index to the latest supported Solr version - IndexUpgrader indexUpgrader = new IndexUpgrader(); - indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory()); - - // set the upgraded reference index as the index to be used for this case - currentVersionIndexDir = newIndexDir; + } else { + // found index that is latest version of Solr and schema + schemaVer = IndexFinder.getCurrentSchemaVersion(); } - - // open currentVersionIndexDir index - - // execute a test query - // if failed, close the upgraded index? + + // open core + try { + // ELTODO figure out the schema version of the core that's being opened + //KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndexDir, schemaVer); + KeywordSearch.getServer().openCoreForCase(context.getCase()); + } catch (Exception ex) { + logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS + if (RuntimeProperties.coreComponentsAreActive()) { + MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage()); + } + } + + // ELTODO execute a test query + // ELTODO if failed, close the upgraded index? } /** @@ -209,6 +233,22 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService /* * Autopsy service providers may not have case-level resources. */ + try { + KeywordSearchResultFactory.BlackboardResultWriter.stopAllWriters(); + /* + * TODO (AUT-2084): The following code + * KeywordSearch.CaseChangeListener gambles that any + * BlackboardResultWriters (SwingWorkers) will complete + * in less than roughly two seconds + */ + Thread.sleep(2000); + KeywordSearch.getServer().closeCore(); + } catch (Exception ex) { + logger.log(Level.SEVERE, String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS + if (RuntimeProperties.coreComponentsAreActive()) { + MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.closeCore.notification.msg"), ex.getMessage()); + } + } } @Override From 6ec2ae379e6b06a104213785a0ac3438eb2283de Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Jan 2017 14:20:37 -0500 Subject: [PATCH 02/12] Added logic to identify and store index's solr and schema versions --- .../autopsy/keywordsearch/Index.java | 93 ++++++++++++++ .../autopsy/keywordsearch/IndexFinder.java | 115 ++++++++++++++++-- .../keywordsearch/SolrSearchService.java | 80 ++++++------ 3 files changed, 238 insertions(+), 50 deletions(-) create mode 100644 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java new file mode 100644 index 0000000000..54977676b1 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java @@ -0,0 +1,93 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2016 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +/** + * This class encapsulates KWS index data. + */ +class Index { + + private String indexPath; + private String schemaVersion; + private String solrVersion; + + Index() { + this.indexPath = ""; + this.solrVersion = ""; + this.schemaVersion = ""; + } + + Index(String indexPath, String solrVersion, String schemaVersion) { + this.indexPath = indexPath; + this.solrVersion = solrVersion; + this.schemaVersion = schemaVersion; + } + + /** + * @return the indexPath + */ + String getIndexPath() { + return indexPath; + } + + /** + * @param indexPath the indexPath to set + */ + void setIndexPath(String indexPath) { + this.indexPath = indexPath; + } + + /** + * @return the schemaVersion + */ + String getSchemaVersion() { + return schemaVersion; + } + + /** + * @param schemaVersion the schemaVersion to set + */ + void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } + + /** + * @return the solrVersion + */ + String getSolrVersion() { + return solrVersion; + } + + /** + * @param solrVersion the solrVersion to set + */ + void setSolrVersion(String solrVersion) { + this.solrVersion = solrVersion; + } + + /** + * @param true if all Index fields are set, false otherwise + */ + boolean isIndexDataPopulated() { + if (!this.indexPath.isEmpty() && !this.solrVersion.isEmpty() && !this.schemaVersion.isEmpty()) { + return true; + } + return false; + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index dd88b79afc..a575b10063 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -63,22 +63,31 @@ class IndexFinder { return CURRENT_SOLR_SCHEMA_VERSION; } - static String findLatestVersionIndexDir(List allIndexes) { + static Index findLatestVersionIndexDir(List allIndexes) { String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; - for (String path : allIndexes) { + for (Index index : allIndexes) { + String path = index.getIndexPath(); if (path.contains(indexFolderName)) { - return path; + return new Index(path, CURRENT_SOLR_VERSION, CURRENT_SOLR_SCHEMA_VERSION); } } - return ""; + return new Index(); } - static String createLatestVersionIndexDir(Case theCase) { + static Index createLatestVersionIndexDir(Case theCase) { String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; // new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index" File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS targetDirPath.mkdirs(); - return targetDirPath.getAbsolutePath(); + return new Index(targetDirPath.getAbsolutePath(), CURRENT_SOLR_VERSION, CURRENT_SOLR_SCHEMA_VERSION); + } + + static Index identifyIndexToUpgrade(List allIndexes) { + // ELTODO not sure what to do when there are multiple old indexes. grab the first one? + // ELTODO need to handle here a case where it is Solr 6 index but not latest schema + // ELTODO figure out the schema version of the "old" index + + return new Index(); } String copyIndexAndConfigSet(Case theCase, String oldIndexDir) throws AutopsyService.AutopsyServiceException { @@ -164,9 +173,9 @@ class IndexFinder { * * @param theCase the case to get index dir for * - * @return List of absolute paths to all found index directories + * @return List of Index objects for each found index directory */ - List findAllIndexDirs(Case theCase) { + List findAllIndexDirs(Case theCase) { ArrayList candidateIndexDirs = new ArrayList<>(); // first find all existing "/ModuleOutput/keywordsearch/data/" folders if (theCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { @@ -212,16 +221,66 @@ class IndexFinder { } // analyze possible index folders - ArrayList indexDirs = new ArrayList<>(); + ArrayList indexes = new ArrayList<>(); for (String path : candidateIndexDirs) { List validIndexPaths = containsValidIndexFolders(path); for (String validPath : validIndexPaths) { - indexDirs.add(convertPathToUNC(validPath)); - // there can be multiple index folders (e.g. current version and "old" version) so keep looking + String solrVersion = getSolrVersionFromIndexPath(validPath); + String schemaVersion = getSchemaVersionFromIndexPath(validPath); + if (!validPath.isEmpty() && !solrVersion.isEmpty() && !schemaVersion.isEmpty()) { + indexes.add(new Index(convertPathToUNC(validPath), solrVersion, schemaVersion)); + // there can be multiple index folders (e.g. current version and "old" version) so keep looking + } } } - return indexDirs; + return indexes; } + + String getSolrVersionFromIndexPath(String path) { + /* NOTE: All of the following paths are valid multi-user index paths: + (Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index + */ + File file = new File(path); + // sanity check - must be "index" folder + if (!file.getName().equals(INDEX_FOLDER_NAME)) { + // invalid index path + return ""; + } + String parentFolderName = file.getParent(); + if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) { + // this is a Solr4 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + return "4"; + } + + // extract Solr version if name matches "solrX_schema_Y" format + return getSolrVersionFromIndexFolderName(parentFolderName); + } + + String getSchemaVersionFromIndexPath(String path) { + /* NOTE: All of the following paths are valid multi-user index paths: + (Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index + */ + File file = new File(path); + // sanity check - must be "index" folder + if (!file.getName().equals(INDEX_FOLDER_NAME)) { + // invalid index path + return ""; + } + String parentFolderName = file.getParent(); + if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) { + // this is a Solr 4 schema 1.8 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + return "1.8"; + } + + // extract schema version if name matches "solrX_schema_Y" format + return getSchemaVersionFromIndexFolderName(parentFolderName); + } String convertPathToUNC(String indexDir) { // ELTODO do we need to do this when searching for old index? @@ -312,8 +371,38 @@ class IndexFinder { * * @return True or false. */ - public static boolean matchesIndexFolderNameStandard(String inputString) { + static boolean matchesIndexFolderNameStandard(String inputString) { Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); return m.find(); } + + /** + * Gets Solr version number if index folder name matches the standard + * + * @param inputString The string to check. + * + * @return Solr version, empty string on error + */ + static String getSolrVersionFromIndexFolderName(String inputString) { + Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); + if (m.find()) { + return m.group(1); + } + return ""; + } + + /** + * Gets Solr schema version number if index folder name matches the standard + * + * @param inputString The string to check. + * + * @return Solr schema version, empty string on error + */ + static String getSchemaVersionFromIndexFolderName(String inputString) { + Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); + if (m.find()) { + return m.group(2); + } + return ""; + } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 62d0f93932..42522d043e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -25,6 +25,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.MissingResourceException; import java.util.logging.Level; +import org.apache.commons.lang.math.NumberUtils; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.openide.util.NbBundle; @@ -160,55 +161,60 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService // do a case subdirectory search to check for the existence and upgrade status of KWS indexes IndexFinder indexFinder = new IndexFinder(); - List indexDirs = indexFinder.findAllIndexDirs(context.getCase()); + List indexes = indexFinder.findAllIndexDirs(context.getCase()); // check if index needs upgrade - String schemaVer; - String currentVersionIndexDir = IndexFinder.findLatestVersionIndexDir(indexDirs); - if (currentVersionIndexDir.isEmpty()) { - if (indexDirs.isEmpty()) { - // new case that doesn't have an existing index. create new index folder - currentVersionIndexDir = IndexFinder.createLatestVersionIndexDir(context.getCase()); - schemaVer = IndexFinder.getCurrentSchemaVersion(); - } else { - // ELTODO not sure what to do when there are multiple old indexes. grab the first one? - // ELTODO need to handle here a case where it is Solr 6 index but not latest schema - // ELTODO figure out the schema version of the "old" index - String oldIndexDir = indexDirs.get(0); + Index currentVersionIndex; + if (indexes.isEmpty()) { + // new case that doesn't have an existing index. create new index folder + currentVersionIndex = IndexFinder.createLatestVersionIndexDir(context.getCase()); + } else { + // check if one of the existing indexes is for latest Solr version and schema + currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes); - if (RuntimeProperties.coreComponentsAreActive()) { - //pop up a message box to indicate the restrictions on adding additional - //text and performing regex searches and give the user the option to decline the upgrade - if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"), - NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"), - KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { - // upgrade declined - throw exception - throw new AutopsyServiceException("Index upgrade was declined by user"); - } + if (!currentVersionIndex.isIndexDataPopulated()) { + // found existing index(es) but none were for latest Solr version and schema version + Index indexToUpgrade = IndexFinder.identifyIndexToUpgrade(indexes); + if (!indexToUpgrade.isIndexDataPopulated()) { + // unable to find index that can be upgraded + throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr"); } - // ELTODO Check for cancellation at whatever points are feasible - - // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ - String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), oldIndexDir); + // get Solr and schema verison of latest index + if (NumberUtils.toDouble(indexToUpgrade.getSolrVersion()) < NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) { + // index needs to be upgraded to latest supported version of Solr - // upgrade the "old" index to the latest supported Solr version - IndexUpgrader indexUpgrader = new IndexUpgrader(); - indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory()); + if (RuntimeProperties.coreComponentsAreActive()) { + //pop up a message box to indicate the restrictions on adding additional + //text and performing regex searches and give the user the option to decline the upgrade + if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"), + NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"), + KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { + // upgrade declined - throw exception + throw new AutopsyServiceException("Index upgrade was declined by user"); + } + } - // set the upgraded reference index as the index to be used for this case - currentVersionIndexDir = newIndexDir; - // schemaVer = ; // ELTODO + // ELTODO Check for cancellation at whatever points are feasible + + // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ + String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade.getIndexPath()); + + // upgrade the "old" index to the latest supported Solr version + IndexUpgrader indexUpgrader = new IndexUpgrader(); + indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory()); + + // set the upgraded index as the index to be used for this case + currentVersionIndex.setIndexPath(newIndexDir); + currentVersionIndex.setSolrVersion(IndexFinder.getCurrentSolrVersion()); + currentVersionIndex.setSchemaVersion(indexToUpgrade.getSchemaVersion()); + } } - } else { - // found index that is latest version of Solr and schema - schemaVer = IndexFinder.getCurrentSchemaVersion(); } // open core try { - // ELTODO figure out the schema version of the core that's being opened - //KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndexDir, schemaVer); + //KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex); KeywordSearch.getServer().openCoreForCase(context.getCase()); } catch (Exception ex) { logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS From d7247add82507c8c9b8a897268f20bcef9f8e222 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Jan 2017 14:48:16 -0500 Subject: [PATCH 03/12] Added logic to pick best existing index as upgrade candidate --- .../autopsy/keywordsearch/IndexFinder.java | 27 +++++++++++++++---- .../keywordsearch/SolrSearchService.java | 14 +++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index a575b10063..6013d6a8f2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -28,6 +28,7 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.math.NumberUtils; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.Logger; @@ -83,11 +84,27 @@ class IndexFinder { } static Index identifyIndexToUpgrade(List allIndexes) { - // ELTODO not sure what to do when there are multiple old indexes. grab the first one? - // ELTODO need to handle here a case where it is Solr 6 index but not latest schema - // ELTODO figure out the schema version of the "old" index - - return new Index(); + /* NOTE: All of the following paths are valid multi-user index paths: + (Solr 4, schema 1.8) X:\Case\ingest1\ModuleOutput\keywordsearch\data\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_2.0\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index + X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index + */ + Index bestCandidateIndex = new Index(); + double solrVerFound = 0.0; + double schemaVerFound = 0.0; + for (Index index : allIndexes) { + // higher Solr version takes priority because it may negate index upgrade + if (NumberUtils.toDouble(index.getSolrVersion()) >= solrVerFound) { + // if same solr version, pick the one with highest schema version + if (NumberUtils.toDouble(index.getSchemaVersion()) > schemaVerFound) { + bestCandidateIndex = index; + solrVerFound = NumberUtils.toDouble(index.getSolrVersion()); + schemaVerFound = NumberUtils.toDouble(index.getSchemaVersion()); + } + } + } + return bestCandidateIndex; } String copyIndexAndConfigSet(Case theCase, String oldIndexDir) throws AutopsyService.AutopsyServiceException { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 42522d043e..76d11df4b4 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -180,10 +180,18 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr"); } - // get Solr and schema verison of latest index - if (NumberUtils.toDouble(indexToUpgrade.getSolrVersion()) < NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) { + double currentSolrVersion = NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion()); + double indexSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion()); + if (indexSolrVersion > currentSolrVersion) { + // oops! + throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr"); + } + else if (indexSolrVersion == currentSolrVersion) { + // latest Solr version but not latest schema. it should be used in read-only mode and not be upgraded + currentVersionIndex = indexToUpgrade; + } + else { // index needs to be upgraded to latest supported version of Solr - if (RuntimeProperties.coreComponentsAreActive()) { //pop up a message box to indicate the restrictions on adding additional //text and performing regex searches and give the user the option to decline the upgrade From 9f9e79dd99290c056c52be952fd2f607807a8a78 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Jan 2017 15:36:52 -0500 Subject: [PATCH 04/12] Multiple bug fixes --- .../autopsy/keywordsearch/IndexFinder.java | 26 ++++++++++--------- .../autopsy/keywordsearch/IndexUpgrader.java | 7 ----- .../keywordsearch/SolrSearchService.java | 2 +- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index 6013d6a8f2..1f2ccff13a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -47,7 +47,8 @@ class IndexFinder { private static final String INDEX_FOLDER_NAME = "index"; private static final String CURRENT_SOLR_VERSION = "6"; private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.0"; - private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr\\d{1,2}_schema_\\d{1,2}.\\d{1,2}$"); + //private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr\\d{1,2}_schema_\\d{1,2}.\\d{1,2}$"); + private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr(\\d{1,2})_schema_(\\d{1,2}\\.\\d{1,2})$"); // If SOLR_HOME environment variable doesn't exist, try these relative paths to find Solr config sets: private static final String RELATIVE_PATH_TO_CONFIG_SET = "autopsy/solr/solr/configsets/"; private static final String RELATIVE_PATH_TO_CONFIG_SET_2 = "release/solr/solr/configsets/"; @@ -69,7 +70,7 @@ class IndexFinder { for (Index index : allIndexes) { String path = index.getIndexPath(); if (path.contains(indexFolderName)) { - return new Index(path, CURRENT_SOLR_VERSION, CURRENT_SOLR_SCHEMA_VERSION); + return index; } } return new Index(); @@ -97,7 +98,7 @@ class IndexFinder { // higher Solr version takes priority because it may negate index upgrade if (NumberUtils.toDouble(index.getSolrVersion()) >= solrVerFound) { // if same solr version, pick the one with highest schema version - if (NumberUtils.toDouble(index.getSchemaVersion()) > schemaVerFound) { + if (NumberUtils.toDouble(index.getSchemaVersion()) >= schemaVerFound) { bestCandidateIndex = index; solrVerFound = NumberUtils.toDouble(index.getSolrVersion()); schemaVerFound = NumberUtils.toDouble(index.getSchemaVersion()); @@ -107,9 +108,9 @@ class IndexFinder { return bestCandidateIndex; } - String copyIndexAndConfigSet(Case theCase, String oldIndexDir) throws AutopsyService.AutopsyServiceException { + String copyIndexAndConfigSet(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException { // Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index - String newIndexDir = createReferenceIndexCopy(theCase, oldIndexDir); + String newIndexDir = createReferenceIndexCopy(theCase, indexToUpgrade); // Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset createReferenceConfigSetCopy(new File(newIndexDir).getParent()); @@ -117,9 +118,10 @@ class IndexFinder { return newIndexDir; } - private String createReferenceIndexCopy(Case theCase, String indexPath) throws AutopsyService.AutopsyServiceException { - logger.log(Level.INFO, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS - String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; + private String createReferenceIndexCopy(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException { + logger.log(Level.INFO, "Creating a reference copy of KWS index in {0} ", indexToUpgrade.getIndexPath()); //NON-NLS + // folder name for the upgraded index should be latest Solr version BUT schema verion of the existing index + String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + indexToUpgrade.getSchemaVersion(); try { // new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index" File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS @@ -128,12 +130,12 @@ class IndexFinder { List contents = getAllContentsInFolder(targetDirPath.getAbsolutePath()); if (!contents.isEmpty()) { // target directory is not empty - logger.log(Level.SEVERE, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS + logger.log(Level.SEVERE, "Creating a reference copy of KWS index in {0} ", indexToUpgrade.getIndexPath()); //NON-NLS throw new AutopsyService.AutopsyServiceException("Directory to store the upgraded index must be empty " + targetDirPath.getAbsolutePath()); } } targetDirPath.mkdirs(); - FileUtils.copyDirectory(new File(indexPath), targetDirPath); + FileUtils.copyDirectory(new File(indexToUpgrade.getIndexPath()), targetDirPath); return targetDirPath.getAbsolutePath(); } catch (Exception ex) { logger.log(Level.SEVERE, "Error occurred while creating a reference copy of keyword search index {0}", ex); //NON-NLS @@ -266,7 +268,7 @@ class IndexFinder { // invalid index path return ""; } - String parentFolderName = file.getParent(); + String parentFolderName = file.getParentFile().getName(); if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) { // this is a Solr4 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index return "4"; @@ -289,7 +291,7 @@ class IndexFinder { // invalid index path return ""; } - String parentFolderName = file.getParent(); + String parentFolderName = file.getParentFile().getName(); if (parentFolderName.equals(KWS_DATA_FOLDER_NAME)) { // this is a Solr 4 schema 1.8 path, e.g. X:\Case\ingest1\ModuleOutput\keywordsearch\data\index return "1.8"; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java index 26c36a5ef3..fdbb7bd6b1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java @@ -24,10 +24,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; -import org.apache.commons.io.FileUtils; import org.openide.modules.InstalledFileLocator; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.autopsy.coreutils.Logger; @@ -41,9 +38,6 @@ public class IndexUpgrader { private static final Logger logger = Logger.getLogger(IndexFinder.class.getName()); private final String JAVA_PATH; - // If SOLR_HOME environment variable doesn't exist, try these relative paths to find Solr config sets: - private static final String RELATIVE_PATH_TO_CONFIG_SET = "autopsy/solr/solr/configsets/"; - private static final String RELATIVE_PATH_TO_CONFIG_SET_2 = "release/solr/solr/configsets/"; IndexUpgrader() { JAVA_PATH = PlatformUtil.getJavaPath(); @@ -83,7 +77,6 @@ public class IndexUpgrader { } } - success = true; // ELTODO remove if (!success) { // delete the new directories new File(newIndexDir).delete(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 76d11df4b4..c0cd719255 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -206,7 +206,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService // ELTODO Check for cancellation at whatever points are feasible // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ - String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade.getIndexPath()); + String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade); // upgrade the "old" index to the latest supported Solr version IndexUpgrader indexUpgrader = new IndexUpgrader(); From 04519bc9bab9676b8711b15871f19a83ab12c662 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Jan 2017 16:09:38 -0500 Subject: [PATCH 05/12] Index upgrade code now mimics TSK.updateDatabaseSchema() --- .../autopsy/keywordsearch/IndexFinder.java | 1 - .../autopsy/keywordsearch/IndexUpgrader.java | 64 +++++++++---------- .../keywordsearch/SolrSearchService.java | 9 +-- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index 1f2ccff13a..70b791f57c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -47,7 +47,6 @@ class IndexFinder { private static final String INDEX_FOLDER_NAME = "index"; private static final String CURRENT_SOLR_VERSION = "6"; private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.0"; - //private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr\\d{1,2}_schema_\\d{1,2}.\\d{1,2}$"); private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr(\\d{1,2})_schema_(\\d{1,2}\\.\\d{1,2})$"); // If SOLR_HOME environment variable doesn't exist, try these relative paths to find Solr config sets: private static final String RELATIVE_PATH_TO_CONFIG_SET = "autopsy/solr/solr/configsets/"; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java index fdbb7bd6b1..85d4381feb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java @@ -24,6 +24,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; +import org.apache.commons.lang.math.NumberUtils; import org.openide.modules.InstalledFileLocator; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.ExecUtil; @@ -43,42 +44,28 @@ public class IndexUpgrader { JAVA_PATH = PlatformUtil.getJavaPath(); } - void performIndexUpgrade(String newIndexDir, String tempResultsDir) throws AutopsyService.AutopsyServiceException { + void performIndexUpgrade(Index indexToUpgrade, String tempResultsDir) throws AutopsyService.AutopsyServiceException { // ELTODO Check for cancellation at whatever points are feasible - + + String newIndexDir = indexToUpgrade.getIndexPath(); + // Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solrX_schema_Y/index File tmpDir = Paths.get(tempResultsDir, "IndexUpgrade").toFile(); //NON-NLS tmpDir.mkdirs(); - - boolean success = true; + + double currentSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion()); try { - // upgrade from Solr 4 to 5. If index is newer than Solr 4 then the upgrade script will throw exception right away. - upgradeSolrIndexVersion4to5(newIndexDir, tempResultsDir); - } catch (AutopsyService.AutopsyServiceException | SecurityException | IOException ex) { - // this may not be an error, for example if index is Solr 5 to begin with - logger.log(Level.SEVERE, "Exception while upgrading keyword search index from Sorl 4 to Solr 5 " + newIndexDir, ex); //NON-NLS - success = false; - } catch (Exception ex2) { - // exceptions thrown by the Solr 4 ot 5 upgrade tool itself may not be an error, for example if index is Solr 5 (or later) to begin with - logger.log(Level.WARNING, "Exception while running Sorl 4 to Solr 5 upgrade tool " + newIndexDir, ex2); //NON-NLS + // upgrade from Solr 4 to 5 + currentSolrVersion = upgradeSolrIndexVersion4to5(currentSolrVersion, newIndexDir, tempResultsDir); + // upgrade from Solr 5 to 6 + currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir); + } catch (Exception ex) { + // catch-all firewall for exceptions thrown by Solr upgrade tools + logger.log(Level.SEVERE, "Exception while running Sorl index upgrade " + newIndexDir, ex); //NON-NLS } - if (success) { - try { - // upgrade from Solr 5 to 6. This one must complete successfully in order to produce a valid Solr 6 index. - upgradeSolrIndexVersion5to6(newIndexDir, tempResultsDir); - } catch (AutopsyService.AutopsyServiceException | SecurityException | IOException ex) { - logger.log(Level.SEVERE, "Exception while upgrading keyword search index from Sorl 5 to Solr 6 " + newIndexDir, ex); //NON-NLS - success = false; - } catch (Exception ex2) { - // exceptions thrown by Solr 5 to 6 upgrade tool itself should always be treated as errors - logger.log(Level.SEVERE, "Exception while running Sorl 5 to Solr 6 upgrade tool " + newIndexDir, ex2); //NON-NLS - success = false; - } - } - - if (!success) { - // delete the new directories + if (currentSolrVersion != NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) { + // upgrade did not complete, delete the new directories new File(newIndexDir).delete(); throw new AutopsyService.AutopsyServiceException("Failed to upgrade existing keyword search index"); } @@ -87,13 +74,17 @@ public class IndexUpgrader { /** * Upgrades Solr index from version 4 to 5. * + * @param currentIndexVersion Current Solr index version * @param solr4IndexPath Full path to Solr v4 index directory * @param tempResultsDir Path to directory where to store log output * - * @return True is index upgraded successfully, false otherwise + * @return The new Solr index version. */ - void upgradeSolrIndexVersion4to5(String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { + double upgradeSolrIndexVersion4to5(double currentIndexVersion, String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { + if (currentIndexVersion != 4.0) { + return currentIndexVersion; + } String outputFileName = "output.txt"; logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 4 to Solr 5", solr4IndexPath); //NON-NLS @@ -128,18 +119,22 @@ public class IndexUpgrader { // alternatively can execute lucene upgrade command from the folder where lucene jars are located // java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index + return 4.0; } /** * Upgrades Solr index from version 5 to 6. * + * @param currentIndexVersion Current Solr index version * @param solr5IndexPath Full path to Solr v5 index directory * @param tempResultsDir Path to directory where to store log output * - * @return True is index upgraded successfully, false otherwise + * @return The new Solr index version. */ - void upgradeSolrIndexVersion5to6(String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { - + double upgradeSolrIndexVersion5to6(double currentIndexVersion, String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException { + if (currentIndexVersion != 5.0) { + return currentIndexVersion; + } String outputFileName = "output.txt"; logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 5 to Solr 6", solr5IndexPath); //NON-NLS @@ -174,6 +169,7 @@ public class IndexUpgrader { // alternatively can execute lucene upgrade command from the folder where lucene jars are located // java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index + return 5.0; } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index c0cd719255..7d42b31298 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -18,10 +18,8 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.nio.file.Paths; import java.util.List; import java.util.MissingResourceException; import java.util.logging.Level; @@ -31,7 +29,6 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; -import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.coreutils.Logger; @@ -205,12 +202,12 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService // ELTODO Check for cancellation at whatever points are feasible - // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ + // Copy the existing index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/ String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), indexToUpgrade); - // upgrade the "old" index to the latest supported Solr version + // upgrade the existing index to the latest supported Solr version IndexUpgrader indexUpgrader = new IndexUpgrader(); - indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory()); + indexUpgrader.performIndexUpgrade(indexToUpgrade, context.getCase().getTempDirectory()); // set the upgraded index as the index to be used for this case currentVersionIndex.setIndexPath(newIndexDir); From d63d1f2457aced981b0ca994212435a6e5f19153 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Jan 2017 16:46:12 -0500 Subject: [PATCH 06/12] Code review --- .../autopsy/keywordsearch/Bundle.properties | 4 ++-- .../autopsy/keywordsearch/Index.java | 2 +- .../autopsy/keywordsearch/IndexFinder.java | 23 ++++++++----------- .../autopsy/keywordsearch/IndexUpgrader.java | 21 ++++++++--------- .../keywordsearch/SolrSearchService.java | 2 +- 5 files changed, 22 insertions(+), 30 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index d6fc2eabed..9242231de9 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -313,5 +313,5 @@ GlobalEditListPanel.keywordDupesSkipped.text={0} keyword was already in the list GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list. GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again. GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again. -SolrSearchService.IndexUpgradeDialog.title=Index Upgrade Required In Order To Open Case -SolrSearchService.IndexUpgradeDialog.msg=Index upgrade can be a very lengthy operation that involves copying existing index and calling third party tools to upgrade it.
Upon upgrade you will be able to see existing keyword search results and perform literal keyword searches on the existing index.
However, you will not be able to add new text to the index or performing regex searches.
You must create a new case and re-run Keyword Search Ingest Module if you want to index new text or performing regex searches.
Do you wish to proceed with the index upgrade? \ No newline at end of file +SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case +SolrSearchService.IndexUpgradeDialog.msg=The text index upgrade can take some time.
When completed, you will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed with the index upgrade? \ No newline at end of file diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java index 54977676b1..67cba2aaf2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index 388df13e9e..932036596e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.keywordsearch; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -41,7 +42,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; class IndexFinder { private static final Logger logger = Logger.getLogger(IndexFinder.class.getName()); - private UNCPathUtilities uncPathUtilities; + private final UNCPathUtilities uncPathUtilities; private static final String KWS_OUTPUT_FOLDER_NAME = "keywordsearch"; private static final String KWS_DATA_FOLDER_NAME = "data"; private static final String INDEX_FOLDER_NAME = "index"; @@ -109,7 +110,7 @@ class IndexFinder { String copyIndexAndConfigSet(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException { // Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index - String newIndexDir = createReferenceIndexCopy(theCase, indexToUpgrade); + String newIndexDir = copyExistingIndex(theCase, indexToUpgrade); // Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset createReferenceConfigSetCopy(new File(newIndexDir).getParent()); @@ -117,8 +118,7 @@ class IndexFinder { return newIndexDir; } - private String createReferenceIndexCopy(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException { - logger.log(Level.INFO, "Creating a reference copy of KWS index in {0} ", indexToUpgrade.getIndexPath()); //NON-NLS + private static String copyExistingIndex(Case theCase, Index indexToUpgrade) throws AutopsyService.AutopsyServiceException { // folder name for the upgraded index should be latest Solr version BUT schema verion of the existing index String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + indexToUpgrade.getSchemaVersion(); try { @@ -129,22 +129,19 @@ class IndexFinder { List contents = getAllContentsInFolder(targetDirPath.getAbsolutePath()); if (!contents.isEmpty()) { // target directory is not empty - logger.log(Level.SEVERE, "Creating a reference copy of KWS index in {0} ", indexToUpgrade.getIndexPath()); //NON-NLS throw new AutopsyService.AutopsyServiceException("Directory to store the upgraded index must be empty " + targetDirPath.getAbsolutePath()); } } targetDirPath.mkdirs(); FileUtils.copyDirectory(new File(indexToUpgrade.getIndexPath()), targetDirPath); return targetDirPath.getAbsolutePath(); - } catch (Exception ex) { - logger.log(Level.SEVERE, "Error occurred while creating a reference copy of keyword search index {0}", ex); //NON-NLS + } catch (AutopsyService.AutopsyServiceException | IOException ex) { throw new AutopsyService.AutopsyServiceException("Error occurred while creating a copy of keyword search index", ex); } } // ELTODO This functionality is NTH: private void createReferenceConfigSetCopy(String indexPath) { - logger.log(Level.INFO, "Creating a reference copy of config set in {0} ", indexPath); //NON-NLS File pathToConfigSet = new File(""); try { // See if there is SOLR_HOME environment variable first @@ -178,14 +175,13 @@ class IndexFinder { if (!pathToConfigSet.getAbsolutePath().isEmpty() && pathToConfigSet.exists()) { FileUtils.copyDirectory(pathToConfigSet, new File(indexPath)); } - } catch (Exception ex) { + } catch (AutopsyService.AutopsyServiceException | IOException ex) { // This feature is a NTH so don't re-throw - logger.log(Level.WARNING, "Error while copying KWS config set to {0}", indexPath); //NON-NLS } } /** - * Find index directory location for the case. This is done via subdirectory + * Find index directory location(s) for the case. This is done via subdirectory * search of all existing "ModuleOutput/node_name/keywordsearch/data/" * folders. * @@ -206,9 +202,8 @@ class IndexFinder { X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index */ - // create a list of all sub-directories + // get a list of all folder's contents List contents = getAllContentsInFolder(theCase.getCaseDirectory()); - if (!contents.isEmpty()) { // decipher "ModuleOutput" directory name from module output path // (e.g. X:\Case\ingest4\ModuleOutput\) because there is no other way to get it... diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java index cbd5646b99..e28f765b32 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,13 +61,14 @@ class IndexUpgrader { currentSolrVersion = upgradeSolrIndexVersion5to6(currentSolrVersion, newIndexDir, tempResultsDir); } catch (Exception ex) { // catch-all firewall for exceptions thrown by Solr upgrade tools - logger.log(Level.SEVERE, "Exception while running Sorl index upgrade " + newIndexDir, ex); //NON-NLS - } - - if (currentSolrVersion != NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) { - // upgrade did not complete, delete the new directories - new File(newIndexDir).delete(); - throw new AutopsyService.AutopsyServiceException("Failed to upgrade existing keyword search index"); + throw new AutopsyService.AutopsyServiceException("Exception while running Solr index upgrade in " + newIndexDir, ex); //NON-NLS + } finally { + if (currentSolrVersion != NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion())) { + // upgrade did not complete, delete the new index directories + if (!new File(newIndexDir).delete()) { + logger.log(Level.SEVERE, "Unable to delete folder {0}", newIndexDir); //NON-NLS + } + } } } @@ -91,14 +92,12 @@ class IndexUpgrader { // find the index upgrade tool final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr4to5IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS if (upgradeToolFolder == null) { - logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool"); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool"); } // full path to index upgrade jar file File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr4IndexUpgrade.jar").toFile(); if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) { - logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file"); } @@ -141,14 +140,12 @@ class IndexUpgrader { // find the index upgrade tool final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr5to6IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS if (upgradeToolFolder == null) { - logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool"); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool"); } // full path to index upgrade jar file File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr5IndexUpgrade.jar").toFile(); if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) { - logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file"); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 7d42b31298..6c02266412 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2017 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); From b84f20715e20ef723e0166f0f836a8851ade6169 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 18 Jan 2017 16:52:49 -0500 Subject: [PATCH 07/12] Bug fix --- .../org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java index e28f765b32..4e8566e934 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java @@ -118,7 +118,7 @@ class IndexUpgrader { // alternatively can execute lucene upgrade command from the folder where lucene jars are located // java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index - return 4.0; + return 5.0; } /** @@ -166,7 +166,7 @@ class IndexUpgrader { // alternatively can execute lucene upgrade command from the folder where lucene jars are located // java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index - return 5.0; + return 6.0; } } From 3133d231e0a7a20a620ae19647aa2d4d9005d020 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 19 Jan 2017 09:18:36 -0500 Subject: [PATCH 08/12] Minor --- .../org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 6c02266412..970abd08ee 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -185,6 +185,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService } else if (indexSolrVersion == currentSolrVersion) { // latest Solr version but not latest schema. it should be used in read-only mode and not be upgraded + // ELTODO do we need to display a message to user here? currentVersionIndex = indexToUpgrade; } else { From 41e5f4702ec704e3fa56a33b08d0c2e6f77ea24a Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 19 Jan 2017 10:01:11 -0500 Subject: [PATCH 09/12] Added message dialog regarding read-only restrictions --- .../autopsy/keywordsearch/Bundle.properties | 4 +++- .../autopsy/keywordsearch/SolrSearchService.java | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 9242231de9..b82b06bbb3 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -314,4 +314,6 @@ GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again. GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again. SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case -SolrSearchService.IndexUpgradeDialog.msg=The text index upgrade can take some time.
When completed, you will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed with the index upgrade? \ No newline at end of file +SolrSearchService.IndexUpgradeDialog.msg=The text index upgrade can take some time.
When completed, you will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed with the index upgrade? +SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only +SolrSearchService.IndexReadOnlyDialog.msg=The text index for this case is read-only
You will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed? diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 970abd08ee..8e2c25b51d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -184,9 +184,18 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr"); } else if (indexSolrVersion == currentSolrVersion) { - // latest Solr version but not latest schema. it should be used in read-only mode and not be upgraded - // ELTODO do we need to display a message to user here? - currentVersionIndex = indexToUpgrade; + // latest Solr version but not latest schema. index should be used in read-only mode and not be upgraded. + if (RuntimeProperties.coreComponentsAreActive()) { + // pop up a message box to indicate the read-only restrictions. + if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.title"), + NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.msg"), + KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { + // case open declined - throw exception + throw new AutopsyServiceException("Case open declined by user"); + } + } + // proceed with case open + currentVersionIndex = indexToUpgrade; } else { // index needs to be upgraded to latest supported version of Solr From 3f38596e7f7a9bd2c4c9af1e7337b12f8d991a27 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 19 Jan 2017 10:58:07 -0500 Subject: [PATCH 10/12] Using upgraded index when opening core --- .../autopsy/keywordsearch/Bundle.properties | 2 +- .../autopsy/keywordsearch/KeywordSearch.java | 2 +- .../autopsy/keywordsearch/Server.java | 39 +++---------------- .../keywordsearch/SolrSearchService.java | 4 +- 4 files changed, 10 insertions(+), 37 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index b82b06bbb3..2bf824795e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -316,4 +316,4 @@ GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. P SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case SolrSearchService.IndexUpgradeDialog.msg=The text index upgrade can take some time.
When completed, you will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed with the index upgrade? SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only -SolrSearchService.IndexReadOnlyDialog.msg=The text index for this case is read-only
You will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed? +SolrSearchService.IndexReadOnlyDialog.msg=The text index for this case is read-only.
You will be able to see existing keyword search results and perform literal keyword searches,
but you will not be able to add new text to the index or perform regex searches. You may instead open the case
with your previous version of this application. Do you wish to proceed? diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java index f491d1696e..7e0b7a9c10 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java @@ -148,7 +148,7 @@ public class KeywordSearch { */ Case openedCase = (Case) evt.getNewValue(); try { - server.openCoreForCase(openedCase); + //server.openCoreForCase(openedCase); } catch (Exception ex) { logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", openedCase.getCaseDirectory()), ex); //NON-NLS if (RuntimeProperties.coreComponentsAreActive()) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index cb3ebb8d21..08b1209d46 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -658,15 +658,15 @@ public class Server { * Creates/opens a Solr core (index) for a case. * * @param theCase The case for which the core is to be created/opened. - * + * @param index The text index that the Solr core should be using. * * @throws KeywordSearchModuleException If an error occurs while * creating/opening the core. */ - void openCoreForCase(Case theCase) throws KeywordSearchModuleException { + void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException { currentCoreLock.writeLock().lock(); try { - currentCore = openCore(theCase); + currentCore = openCore(theCase, index); serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED); } finally { currentCoreLock.writeLock().unlock(); @@ -709,32 +709,6 @@ public class Server { } } - /** - * Get index dir location for the case - * - * @param theCase the case to get index dir for - * - * @return absolute path to index dir - */ - String geCoreDataDirPath(Case theCase) { - // ELTODO this method is going to be removed - String indexDir = theCase.getModuleDirectory() + File.separator + "keywordsearch" + File.separator + "data"; //NON-NLS - if (uncPathUtilities != null) { - // if we can check for UNC paths, do so, otherwise just return the indexDir - String result = uncPathUtilities.mappedDriveToUNC(indexDir); - if (result == null) { - uncPathUtilities.rescanDrives(); - result = uncPathUtilities.mappedDriveToUNC(indexDir); - } - if (result == null) { - return indexDir; - } - return result; - } - return indexDir; - } - - /** * ** end single-case specific methods *** */ @@ -742,15 +716,14 @@ public class Server { * Creates/opens a Solr core (index) for a case. * * @param theCase The case for which the core is to be created/opened. + * @param index The text index that the Solr core should be using. * * @return An object representing the created/opened core. * * @throws KeywordSearchModuleException If an error occurs while * creating/opening the core. */ - private Core openCore(Case theCase) throws KeywordSearchModuleException { - - // ELTODO REMOVE String indexDir = findLatestVersionIndexDir(Case.getCurrentCase()); // ELTODO + private Core openCore(Case theCase, Index index) throws KeywordSearchModuleException { try { if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) { @@ -766,7 +739,7 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex); } - String dataDir = geCoreDataDirPath(theCase); + String dataDir = new File(index.getIndexPath()).getParent(); // "data dir" is the parent of the index directory // ELTODO make sure this is UNC String coreName = theCase.getTextIndexName(); return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType()); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 8e2c25b51d..6f4d37b252 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -229,8 +229,8 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService // open core try { - //KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex); - KeywordSearch.getServer().openCoreForCase(context.getCase()); + KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex); + //KeywordSearch.getServer().openCoreForCase(context.getCase()); } catch (Exception ex) { logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS if (RuntimeProperties.coreComponentsAreActive()) { From 0985c6879ab64764248b8916807407ed7bc70ba1 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 19 Jan 2017 14:46:02 -0500 Subject: [PATCH 11/12] Cores are open via openCaseResources(). Removed Case listeners --- .../sleuthkit/autopsy/casemodule/Case.java | 4 +- .../autopsy/keywordsearch/Installer.java | 1 - .../autopsy/keywordsearch/KeywordSearch.java | 53 ------------------- .../autopsy/keywordsearch/Server.java | 17 +++--- .../keywordsearch/SolrSearchService.java | 1 - 5 files changed, 11 insertions(+), 65 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 6410a269ce..cd984c2155 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1489,9 +1489,7 @@ public class Case implements SleuthkitCase.ErrorObserver { for (AutopsyService service : Lookup.getDefault().lookupAll(AutopsyService.class)) { try { serviceName = service.getServiceName(); - if (!serviceName.equals("Solr Keyword Search Service")) { - service.openCaseResources(context); - } + service.openCaseResources(context); } catch (AutopsyService.AutopsyServiceException ex) { Case.logger.log(Level.SEVERE, String.format("%s service failed to open case resources", serviceName), ex); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java index 7127faf2d5..8119015320 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java @@ -22,7 +22,6 @@ import java.util.logging.Level; import org.openide.modules.ModuleInstall; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; -import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.keywordsearch.Server.SolrServerNoPortException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java index 7e0b7a9c10..5d5c0847bb 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.io.IOException; @@ -28,10 +27,8 @@ import java.util.logging.Logger; import java.util.logging.SimpleFormatter; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.keywordsearch.KeywordSearchResultFactory.BlackboardResultWriter; /** * Wrapper over KeywordSearch Solr server singleton. The class also provides @@ -109,54 +106,4 @@ public class KeywordSearch { MessageNotifyUtil.MessageType.ERROR); } } - - /** - * Listener to create/open and close Solr cores when cases are - * created/opened and closed. - */ - static class CaseChangeListener implements PropertyChangeListener { - - @Override - public void propertyChange(PropertyChangeEvent evt) { - if (evt.getPropertyName().equals(Case.Events.CURRENT_CASE.toString())) { - if (null != evt.getOldValue()) { - /* - * A case is being closed. - */ - Case closedCase = (Case) evt.getOldValue(); - try { - BlackboardResultWriter.stopAllWriters(); - /* - * TODO (AUT-2084): The following code - * KeywordSearch.CaseChangeListener gambles that any - * BlackboardResultWriters (SwingWorkers) will complete - * in less than roughly two seconds - */ - Thread.sleep(2000); - server.closeCore(); - } catch (Exception ex) { - logger.log(Level.SEVERE, String.format("Failed to close core for %s", closedCase.getCaseDirectory()), ex); //NON-NLS - if (RuntimeProperties.coreComponentsAreActive()) { - MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.closeCore.notification.msg"), ex.getMessage()); - } - } - } - - if (null != evt.getNewValue()) { - /* - * A case is being created/opened. - */ - Case openedCase = (Case) evt.getNewValue(); - try { - //server.openCoreForCase(openedCase); - } catch (Exception ex) { - logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", openedCase.getCaseDirectory()), ex); //NON-NLS - if (RuntimeProperties.coreComponentsAreActive()) { - MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage()); - } - } - } - } - } - } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 08b1209d46..545d7c7cf9 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -739,9 +739,8 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex); } - String dataDir = new File(index.getIndexPath()).getParent(); // "data dir" is the parent of the index directory // ELTODO make sure this is UNC String coreName = theCase.getTextIndexName(); - return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType()); + return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, index, theCase.getCaseType()); } /** @@ -1088,7 +1087,7 @@ public class Server { * Creates/opens a Solr core (index) for a case. * * @param coreName The core name. - * @param dataDir The data directory for the core. + * @param index The text index object for the core. * @param caseType The type of the case (single-user or multi-user) for * which the core is being created/opened. * @@ -1097,9 +1096,11 @@ public class Server { * @throws KeywordSearchModuleException If an error occurs while * creating/opening the core. */ - private Core openCore(String coreName, File dataDir, CaseType caseType) throws KeywordSearchModuleException { + private Core openCore(String coreName, Index index, CaseType caseType) throws KeywordSearchModuleException { try { + + File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory // ELTODO make sure this is UNC if (!dataDir.exists()) { dataDir.mkdirs(); } @@ -1142,8 +1143,7 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg")); } - // ELTODO set solr and schema version of the core that is being loaded. Make that available via API. - return new Core(coreName, caseType); + return new Core(coreName, caseType, index); } catch (SolrServerException | SolrException | IOException ex) { throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex); @@ -1208,14 +1208,17 @@ public class Server { private final String name; private final CaseType caseType; + + private final Index textIndex; // the server to access a core needs to be built from a URL with the // core in it, and is only good for core-specific operations private final HttpSolrClient solrCore; - private Core(String name, CaseType caseType) { + private Core(String name, CaseType caseType, Index index) { this.name = name; this.caseType = caseType; + this.textIndex = index; this.solrCore = new Builder(currentSolrServer.getBaseURL() + "/" + name).build(); //NON-NLS diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 6f4d37b252..4f6f4525c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -230,7 +230,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService // open core try { KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex); - //KeywordSearch.getServer().openCoreForCase(context.getCase()); } catch (Exception ex) { logger.log(Level.SEVERE, String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex); //NON-NLS if (RuntimeProperties.coreComponentsAreActive()) { From 18723478e6b8a1c92ff16c483e0ea3f0cba80ec1 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Thu, 19 Jan 2017 14:56:09 -0500 Subject: [PATCH 12/12] Resolved some TODOs --- .../src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java | 5 ++--- .../src/org/sleuthkit/autopsy/keywordsearch/Server.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java index 932036596e..9502da5c76 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java @@ -153,7 +153,7 @@ class IndexFinder { // if there is no SOLR_HOME: // this will only work for Windows OS if (!PlatformUtil.isWindowsOS()) { - throw new AutopsyService.AutopsyServiceException("ELTODO"); + throw new AutopsyService.AutopsyServiceException("Creating a reference config set copy is currently a Windows-only feature"); } // config set should be located in "C:/some/directory/AutopsyXYZ/autopsy/solr/solr/configsets/" pathToConfigSet = Paths.get(System.getProperty("user.dir"), RELATIVE_PATH_TO_CONFIG_SET).toFile(); @@ -163,7 +163,7 @@ class IndexFinder { if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) { logger.log(Level.WARNING, "Unable to locate KWS config set in order to create a reference copy"); //NON-NLS return; - // ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("ELTODO"); + // ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("Unable to locate the config set"); } } } @@ -296,7 +296,6 @@ class IndexFinder { } String convertPathToUNC(String indexDir) { - // ELTODO do we need to do this when searching for old index? if (uncPathUtilities == null) { return indexDir; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 545d7c7cf9..2c8d44c5b2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -1100,7 +1100,7 @@ public class Server { try { - File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory // ELTODO make sure this is UNC + File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory if (!dataDir.exists()) { dataDir.mkdirs(); }