Merge pull request #2472 from eugene7646/index_upgrade_4

Solr 4 to 6 index upgrade

Core/src/org/sleuthkit/autopsy/casemodule/Case.java

@@ -663,6 +663,18 @@ public class Case implements SleuthkitCase.ErrorObserver {
         return getCaseMetadata().getTextIndexName();
     }
     
+    /**
+     * Sets the name of the keyword search index for the case.
+     *
+     * @param indexName The index name.
+     *
+     * @throws
+     * org.sleuthkit.autopsy.casemodule.CaseMetadata.CaseMetadataException
+     */
+    public void setTextIndexName(String indexName) throws CaseMetadataException {
+        caseMetadata.setTextIndexName(indexName);
+    }
+
     /**
      * Queries whether or not the case has data, i.e., whether or not at least
      * one data source has been added to the case.
@@ -984,19 +996,17 @@ public class Case implements SleuthkitCase.ErrorObserver {
         }
 
         /*
-         * Sanitize the case name, create a unique keyword search index name,
+         * Sanitize the case name, and create a standard (single-user) or 
-         * and create a standard (single-user) or unique (multi-user) case
+         * unique (multi-user) case database name.
-         * database name.
          */
         String santizedCaseName = sanitizeCaseName(caseName);
         SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
         Date date = new Date();
-        String indexName = santizedCaseName + "_" + dateFormat.format(date);
         String dbName = null;
         if (caseType == CaseType.SINGLE_USER_CASE) {
             dbName = caseDir + File.separator + "autopsy.db"; //NON-NLS
         } else if (caseType == CaseType.MULTI_USER_CASE) {
-            dbName = indexName;
+            dbName = santizedCaseName + "_" + dateFormat.format(date);
         }
         
         try{
@@ -1037,7 +1047,7 @@ public class Case implements SleuthkitCase.ErrorObserver {
              */
             CaseMetadata metadata;
             try {
-                metadata = new CaseMetadata(caseDir, caseType, caseName, caseNumber, examiner, dbName, indexName);
+                metadata = new CaseMetadata(caseDir, caseType, caseName, caseNumber, examiner, dbName);
             } catch (CaseMetadataException ex) {
                 throw new CaseActionException(Bundle.Case_creationException(), ex);
             }

Core/src/org/sleuthkit/autopsy/casemodule/CaseMetadata.java

@@ -102,19 +102,17 @@ public final class CaseMetadata {
      * @param caseDatabase      For a single-user case, the full path to the
      *                          case database file. For a multi-user case, the
      *                          case database name.
-     * @param caseTextIndexName The text index name.
      *
      * @throws CaseMetadataException If the new case metadata file cannot be
      *                               created.
      */
-    CaseMetadata(String caseDirectory, Case.CaseType caseType, String caseName, String caseNumber, String examiner, String caseDatabase, String caseTextIndexName) throws CaseMetadataException {
+    CaseMetadata(String caseDirectory, Case.CaseType caseType, String caseName, String caseNumber, String examiner, String caseDatabase) throws CaseMetadataException {
         metadataFilePath = Paths.get(caseDirectory, caseName + FILE_EXTENSION);
         this.caseType = caseType;
         this.caseName = caseName;
         this.caseNumber = caseNumber;
         this.examiner = examiner;
         this.caseDatabase = caseDatabase;
-        this.textIndexName = caseTextIndexName;
         createdByVersion = Version.getVersion();
         createdDate = CaseMetadata.DATE_FORMAT.format(new Date());
         writeToFile();
@@ -187,6 +185,22 @@ public final class CaseMetadata {
         }
     }
     
+    /**
+     * Sets the text index name.
+     *
+     * @param caseTextIndexName The text index name.
+     */
+    void setTextIndexName(String caseTextIndexName) throws CaseMetadataException {
+        String oldIndexName = caseTextIndexName;
+        this.textIndexName = caseTextIndexName;
+        try {
+            writeToFile();
+        } catch (CaseMetadataException ex) {
+            this.textIndexName = oldIndexName;
+            throw ex;
+        }
+    }    
+
     /**
      * Gets the case number.
      *

Core/src/org/sleuthkit/autopsy/casemodule/SingleUserCaseConverter.java

@@ -176,7 +176,6 @@ public class SingleUserCaseConverter {
         SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); //NON-NLS
         Date date = new Date();
         String dbName = Case.sanitizeCaseName(icd.getNewCaseName()) + "_" + dateFormat.format(date); //NON-NLS
-        String solrName = dbName;
         icd.setPostgreSQLDbName(dbName);
 
         // Copy items to new hostname folder structure
@@ -197,7 +196,7 @@ public class SingleUserCaseConverter {
                 icd.getNewCaseName(),
                 oldCaseMetadata.getCaseNumber(),
                 oldCaseMetadata.getExaminer(),
-                dbName, solrName);
+                dbName);
         // Set created date. This calls writefile, no need to call it again
         newCaseMetadata.setCreatedDate(oldCaseMetadata.getCreatedDate());
         newCaseMetadata.setCreatedByVersion(oldCaseMetadata.getCreatedByVersion());

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties

@@ -313,6 +313,7 @@ GlobalEditListPanel.keywordDupesSkipped.text={0} keyword was already in the list
 GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list.
 GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again.
 GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again.
+SolrSearchService.ServiceName=Solr Keyword Search Service
 SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case
 SolrSearchService.IndexUpgradeDialog.msg=<html>The text index upgrade can take some time. <br />When completed, you will be able to see existing keyword search results and perform literal keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application. Do you wish to proceed with the index upgrade?</html>
 SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java

@@ -23,20 +23,16 @@ package org.sleuthkit.autopsy.keywordsearch;
  */
 class Index {
     
-    private String indexPath;
+    private final String indexPath;
-    private String schemaVersion;
+    private final String schemaVersion;
-    private String solrVersion;
+    private final String solrVersion;
-    
+    private boolean newIndex;
-    Index() {
-        this.indexPath = "";
-        this.solrVersion = "";
-        this.schemaVersion = "";
-    }
     
     Index(String indexPath, String solrVersion, String schemaVersion) {
         this.indexPath = indexPath;
         this.solrVersion = solrVersion;
         this.schemaVersion = schemaVersion;
+        newIndex = false;
     }   
 
     /**
@@ -46,13 +42,6 @@ class Index {
         return indexPath;
     }
 
-    /**
-     * @param indexPath the indexPath to set
-     */
-    void setIndexPath(String indexPath) {
-        this.indexPath = indexPath;
-    }
-
     /**
      * @return the schemaVersion
      */
@@ -60,13 +49,6 @@ class Index {
         return schemaVersion;
     }
 
-    /**
-     * @param schemaVersion the schemaVersion to set
-     */
-    void setSchemaVersion(String schemaVersion) {
-        this.schemaVersion = schemaVersion;
-    }
-
     /**
      * @return the solrVersion
      */
@@ -75,19 +57,16 @@ class Index {
     }
 
     /**
-     * @param solrVersion the solrVersion to set
+     * @return the newIndex
      */
-    void setSolrVersion(String solrVersion) {
+    boolean isNewIndex() {
-        this.solrVersion = solrVersion;
+        return newIndex;
     }
 
     /**
-     * @param true if all Index fields are set, false otherwise
+     * @param newIndex the newIndex to set
      */
-    boolean isIndexDataPopulated() {
+    void setNewIndex(boolean newIndex) {
-        if (!this.indexPath.isEmpty() && !this.solrVersion.isEmpty() && !this.schemaVersion.isEmpty()) {
+        this.newIndex = newIndex;
-            return true;
-        }
-        return false;
     }
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java

@@ -73,7 +73,7 @@ class IndexFinder {
                 return index;
             }
         }
-        return new Index();
+        return null;
     }
     
     static Index createLatestVersionIndexDir(Case theCase) {
@@ -91,7 +91,7 @@ class IndexFinder {
             X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index
             X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index
          */
-        Index bestCandidateIndex = new Index();
+        Index bestCandidateIndex = null;
         double solrVerFound = 0.0;
         double schemaVerFound = 0.0;
         for (Index index : allIndexes) {

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java

@@ -36,9 +36,11 @@ import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
@@ -63,6 +65,7 @@ import org.openide.modules.Places;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.Case.CaseType;
+import org.sleuthkit.autopsy.casemodule.CaseMetadata;
 import org.sleuthkit.autopsy.core.UserPreferences;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.coreutils.ModuleSettings;
@@ -188,8 +191,7 @@ public class Server {
     private int currentSolrServerPort = 0;
     private int currentSolrStopPort = 0;
     private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT);
-    private UNCPathUtilities uncPathUtilities = null;
+   private static final String SOLR = "solr";
-    private static final String SOLR = "solr";
     private static final String CORE_PROPERTIES = "core.properties";
 
     public enum CORE_EVT_STATES {
@@ -257,7 +259,6 @@ public class Server {
             }
         }
         currentCoreLock = new ReentrantReadWriteLock(true);
-        uncPathUtilities = new UNCPathUtilities();
 
         logger.log(Level.INFO, "Created Server instance using Java at {0}", javaHome); //NON-NLS
     }
@@ -667,6 +668,14 @@ public class Server {
         currentCoreLock.writeLock().lock();
         try {
             currentCore = openCore(theCase, index);
+            
+            try {
+                // execute a test query. if it fails, an exception will be thrown
+                queryNumIndexedFiles();
+            } catch (NoOpenCoreException ex) {
+                throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
+            }
+            
             serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
         } finally {
             currentCoreLock.writeLock().unlock();
@@ -739,8 +748,142 @@ public class Server {
             throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex);
         }
 
-        String coreName = theCase.getTextIndexName();
+        try {
-        return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, index, theCase.getCaseType());
+            String coreName = getCoreName(index, theCase);
+
+            File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
+            if (!dataDir.exists()) {
+                dataDir.mkdirs();
+            }
+
+            if (!this.isRunning()) {
+                logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
+                throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
+            }
+
+            if (!coreIsLoaded(coreName)) {
+                /*
+                 * The core either does not exist or it is not loaded. Make a
+                 * request that will cause the core to be created if it does not
+                 * exist or loaded if it already exists.
+                 */
+
+                // In single user mode, if there is a core.properties file already,
+                // we've hit a solr bug. Compensate by deleting it.
+                if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
+                    Path corePropertiesFile = Paths.get(solrFolder.toString(), SOLR, coreName, CORE_PROPERTIES);
+                    if (corePropertiesFile.toFile().exists()) {
+                        try {
+                            corePropertiesFile.toFile().delete();
+                        } catch (Exception ex) {
+                            logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
+                        }
+                    }
+                }
+
+                CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
+                createCoreRequest.setDataDir(dataDir.getAbsolutePath());
+                createCoreRequest.setCoreName(coreName);
+                createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
+                createCoreRequest.setIsLoadOnStartup(false);
+                createCoreRequest.setIsTransient(true);
+                currentSolrServer.request(createCoreRequest);
+            }
+
+            if (!coreIndexFolderExists(coreName)) {
+                throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
+            }
+
+            return new Core(coreName, theCase.getCaseType(), index);
+
+        } catch (SolrServerException | SolrException | IOException | CaseMetadata.CaseMetadataException ex) {
+            throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
+        }
+    }
+    
+    /**
+     * Get or create a sanitized Solr core name. Stores the core name if needed.
+     *
+     * @param index   Index object
+     * @param theCase Case object
+     *
+     * @return The sanitized Solr core name
+     */
+    private String getCoreName(Index index, Case theCase) throws CaseMetadata.CaseMetadataException {
+        String coreName = "";
+        if (index.isNewIndex()) {
+            // come up with a new core name
+            coreName = createCoreName(theCase.getName());
+            // store the new core name
+            theCase.setTextIndexName(coreName);
+        } else {
+            // get core name
+            coreName = theCase.getTextIndexName();
+            if (coreName.isEmpty()) {
+                // come up with a new core name
+                coreName = createCoreName(theCase.getName());
+                // store the new core name
+                theCase.setTextIndexName(coreName);
+            }
+        }
+        return coreName;
+    }
+    
+    /**
+     * Create and sanitize a core name.
+     *
+     * @param caseName Case name
+     *
+     * @return The sanitized Solr core name
+     */
+    private String createCoreName(String caseName) {
+        if (caseName.isEmpty()) {
+            caseName = DEFAULT_CORE_NAME;
+        }
+        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
+        Date date = new Date();
+        String coreName = caseName + "_" + dateFormat.format(date);
+        return sanitizeCoreName(coreName);
+    }
+    
+    /**
+     * Sanitizes the case name for Solr cores.
+     *
+     * Solr:
+     * http://stackoverflow.com/questions/29977519/what-makes-an-invalid-core-name
+     * may not be / \ :
+     * Starting Solr6: core names must consist entirely of periods, underscores, hyphens, and alphanumerics as well not start with a hyphen. may not contain space characters.
+     *
+     * @param coreName A candidate core name.
+     *
+     * @return The sanitized core name.
+     */
+    static private String sanitizeCoreName(String coreName) {
+
+        String result;
+
+        // Remove all non-ASCII characters
+        result = coreName.replaceAll("[^\\p{ASCII}]", "_"); //NON-NLS
+
+        // Remove all control characters
+        result = result.replaceAll("[\\p{Cntrl}]", "_"); //NON-NLS
+
+        // Remove spaces / \ : ? ' "
+        result = result.replaceAll("[ /?:'\"\\\\]", "_"); //NON-NLS
+        
+        // Make it all lowercase
+        result = result.toLowerCase();
+
+        // Must not start with hyphen
+        if (result.length() > 0 && !(Character.isLetter(result.codePointAt(0))) && !(result.codePointAt(0) == '-')) {
+            result = "_" + result;
+        }
+
+        if (result.isEmpty()) {
+            result = DEFAULT_CORE_NAME;
+        }
+
+        return result;
     }
 
     /**
@@ -1083,73 +1226,6 @@ public class Server {
         return Long.toString(parentID) + Server.CHUNK_ID_SEPARATOR + Integer.toString(childID);
     }
 
-    /**
-     * Creates/opens a Solr core (index) for a case.
-     *
-     * @param coreName The core name.
-     * @param index    The text index object for the core.
-     * @param caseType The type of the case (single-user or multi-user) for
-     *                 which the core is being created/opened.
-     *
-     * @return An object representing the created/opened core.
-     *
-     * @throws KeywordSearchModuleException If an error occurs while
-     *                                      creating/opening the core.
-     */
-    private Core openCore(String coreName, Index index, CaseType caseType) throws KeywordSearchModuleException {
-
-        try {
-            
-            File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
-            if (!dataDir.exists()) {
-                dataDir.mkdirs();
-            }
-
-            if (!this.isRunning()) {
-                logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
-                throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
-            }
-
-            if (!coreIsLoaded(coreName)) {
-                /*
-                 * The core either does not exist or it is not loaded. Make a
-                 * request that will cause the core to be created if it does not
-                 * exist or loaded if it already exists.
-                 */
-
-                // In single user mode, if there is a core.properties file already,
-                // we've hit a solr bug. Compensate by deleting it.
-                if (caseType == CaseType.SINGLE_USER_CASE) {
-                    Path corePropertiesFile = Paths.get(solrFolder.toString(), SOLR, coreName, CORE_PROPERTIES);
-                    if (corePropertiesFile.toFile().exists()) {
-                        try {
-                            corePropertiesFile.toFile().delete();
-                        } catch (Exception ex) {
-                            logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
-                        }
-                    }
-                }
-
-                CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
-                createCoreRequest.setDataDir(dataDir.getAbsolutePath());
-                createCoreRequest.setCoreName(coreName);
-                createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
-                createCoreRequest.setIsLoadOnStartup(false);
-                createCoreRequest.setIsTransient(true);
-                currentSolrServer.request(createCoreRequest);
-            }
-
-            if (!coreIndexFolderExists(coreName)) {
-                throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
-            }
-
-            return new Core(coreName, caseType, index);
-
-        } catch (SolrServerException | SolrException | IOException ex) {
-            throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
-        }
-    }
-
     /**
      * Attempts to connect to the given Solr server.
      *

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java

@@ -52,7 +52,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
     private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS
     private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS
     private static final int IS_REACHABLE_TIMEOUT_MS = 1000;
-    private static final String SERVICE_NAME = "Solr Keyword Search Service";
 
     ArtifactTextExtractor extractor = new ArtifactTextExtractor();
 
@@ -165,14 +164,14 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
         if (indexes.isEmpty()) {
             // new case that doesn't have an existing index. create new index folder
             currentVersionIndex = IndexFinder.createLatestVersionIndexDir(context.getCase());
+            currentVersionIndex.setNewIndex(true);
         } else {
             // check if one of the existing indexes is for latest Solr version and schema
             currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes);
-
+            if (currentVersionIndex == null) {
-            if (!currentVersionIndex.isIndexDataPopulated()) {
                 // found existing index(es) but none were for latest Solr version and schema version
                 Index indexToUpgrade = IndexFinder.identifyIndexToUpgrade(indexes);
-                if (!indexToUpgrade.isIndexDataPopulated()) {
+                if (indexToUpgrade == null) {
                     // unable to find index that can be upgraded
                     throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
                 }
@@ -181,7 +180,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
                 double indexSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
                 if (indexSolrVersion > currentSolrVersion) {
                     // oops!
-                    throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
+                    throw new AutopsyServiceException("Unable to find index to use for Case open");
                 } 
                 else if (indexSolrVersion == currentSolrVersion) {
                     // latest Solr version but not latest schema. index should be used in read-only mode and not be upgraded.
@@ -220,9 +219,8 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
                     indexUpgrader.performIndexUpgrade(indexToUpgrade, context.getCase().getTempDirectory());
 
                     // set the upgraded index as the index to be used for this case
-                    currentVersionIndex.setIndexPath(newIndexDir);
+                    currentVersionIndex = new Index(newIndexDir, IndexFinder.getCurrentSolrVersion(), indexToUpgrade.getSchemaVersion());
-                    currentVersionIndex.setSolrVersion(IndexFinder.getCurrentSolrVersion());
+                    currentVersionIndex.setNewIndex(true);
-                    currentVersionIndex.setSchemaVersion(indexToUpgrade.getSchemaVersion());
                 }
             }
         }
@@ -236,9 +234,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
                 MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage());
             }
         }
-
-        // ELTODO execute a test query
-        // ELTODO if failed, close the upgraded index?
     }
 
     /**
@@ -273,6 +268,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
 
     @Override
     public String getServiceName() {
-        return SERVICE_NAME;
+        return NbBundle.getMessage(this.getClass(), "SolrSearchService.ServiceName");
     }
 }