mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 02:07:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2472 from eugene7646/index_upgrade_4

Browse Source 
Solr 4 to 6 index upgrade
This commit is contained in:
Richard Cordovano 2017-01-20 16:35:20 -05:00 committed by GitHub
commit dd5a03e8d0
8 changed files with 206 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Core/src/org/sleuthkit/autopsy/casemodule/Case.java
View File

@ -663,6 +663,18 @@ public class Case implements SleuthkitCase.ErrorObserver {
return getCaseMetadata().getTextIndexName();
}
/**
* Sets the name of the keyword search index for the case.
*
* @param indexName The index name.
*
* @throws
* org.sleuthkit.autopsy.casemodule.CaseMetadata.CaseMetadataException
*/
public void setTextIndexName(String indexName) throws CaseMetadataException {
caseMetadata.setTextIndexName(indexName);
}
/**
* Queries whether or not the case has data, i.e., whether or not at least
* one data source has been added to the case.
@ -984,19 +996,17 @@ public class Case implements SleuthkitCase.ErrorObserver {
}
/*
* Sanitize the case name, create a unique keyword search index name,
* and create a standard (single-user) or unique (multi-user) case
* database name.
* Sanitize the case name, and create a standard (single-user) or
* unique (multi-user) case database name.
*/
String santizedCaseName = sanitizeCaseName(caseName);
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
Date date = new Date();
String indexName = santizedCaseName + "_" + dateFormat.format(date);
String dbName = null;
if (caseType == CaseType.SINGLE_USER_CASE) {
dbName = caseDir + File.separator + "autopsy.db"; //NON-NLS
} else if (caseType == CaseType.MULTI_USER_CASE) {
dbName = indexName;
dbName = santizedCaseName + "_" + dateFormat.format(date);
}
try{
@ -1037,7 +1047,7 @@ public class Case implements SleuthkitCase.ErrorObserver {
*/
CaseMetadata metadata;
try {
metadata = new CaseMetadata(caseDir, caseType, caseName, caseNumber, examiner, dbName, indexName);
metadata = new CaseMetadata(caseDir, caseType, caseName, caseNumber, examiner, dbName);
} catch (CaseMetadataException ex) {
throw new CaseActionException(Bundle.Case_creationException(), ex);
}

20
Core/src/org/sleuthkit/autopsy/casemodule/CaseMetadata.java
View File

@ -102,19 +102,17 @@ public final class CaseMetadata {
* @param caseDatabase For a single-user case, the full path to the
* case database file. For a multi-user case, the
* case database name.
* @param caseTextIndexName The text index name.
*
* @throws CaseMetadataException If the new case metadata file cannot be
* created.
*/
CaseMetadata(String caseDirectory, Case.CaseType caseType, String caseName, String caseNumber, String examiner, String caseDatabase, String caseTextIndexName) throws CaseMetadataException {
CaseMetadata(String caseDirectory, Case.CaseType caseType, String caseName, String caseNumber, String examiner, String caseDatabase) throws CaseMetadataException {
metadataFilePath = Paths.get(caseDirectory, caseName + FILE_EXTENSION);
this.caseType = caseType;
this.caseName = caseName;
this.caseNumber = caseNumber;
this.examiner = examiner;
this.caseDatabase = caseDatabase;
this.textIndexName = caseTextIndexName;
createdByVersion = Version.getVersion();
createdDate = CaseMetadata.DATE_FORMAT.format(new Date());
writeToFile();
@ -187,6 +185,22 @@ public final class CaseMetadata {
}
}
/**
* Sets the text index name.
*
* @param caseTextIndexName The text index name.
*/
void setTextIndexName(String caseTextIndexName) throws CaseMetadataException {
String oldIndexName = caseTextIndexName;
this.textIndexName = caseTextIndexName;
try {
writeToFile();
} catch (CaseMetadataException ex) {
this.textIndexName = oldIndexName;
throw ex;
}
}
/**
* Gets the case number.
*

3
Core/src/org/sleuthkit/autopsy/casemodule/SingleUserCaseConverter.java
View File

@ -176,7 +176,6 @@ public class SingleUserCaseConverter {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); //NON-NLS
Date date = new Date();
String dbName = Case.sanitizeCaseName(icd.getNewCaseName()) + "_" + dateFormat.format(date); //NON-NLS
String solrName = dbName;
icd.setPostgreSQLDbName(dbName);
// Copy items to new hostname folder structure
@ -197,7 +196,7 @@ public class SingleUserCaseConverter {
icd.getNewCaseName(),
oldCaseMetadata.getCaseNumber(),
oldCaseMetadata.getExaminer(),
dbName, solrName);
dbName);
// Set created date. This calls writefile, no need to call it again
newCaseMetadata.setCreatedDate(oldCaseMetadata.getCreatedDate());
newCaseMetadata.setCreatedByVersion(oldCaseMetadata.getCreatedByVersion());

1
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
View File

@ -313,6 +313,7 @@ GlobalEditListPanel.keywordDupesSkipped.text={0} keyword was already in the list
GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list.
GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again.
GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again.
SolrSearchService.ServiceName=Solr Keyword Search Service
SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case
SolrSearchService.IndexUpgradeDialog.msg=<html>The text index upgrade can take some time. <br />When completed, you will be able to see existing keyword search results and perform literal keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application. Do you wish to proceed with the index upgrade?</html>
SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only

43
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java
View File

@ -23,20 +23,16 @@ package org.sleuthkit.autopsy.keywordsearch;
*/
class Index {
private String indexPath;
private String schemaVersion;
private String solrVersion;
Index() {
this.indexPath = "";
this.solrVersion = "";
this.schemaVersion = "";
}
private final String indexPath;
private final String schemaVersion;
private final String solrVersion;
private boolean newIndex;
Index(String indexPath, String solrVersion, String schemaVersion) {
this.indexPath = indexPath;
this.solrVersion = solrVersion;
this.schemaVersion = schemaVersion;
newIndex = false;
}
/**
@ -46,13 +42,6 @@ class Index {
return indexPath;
}
/**
* @param indexPath the indexPath to set
*/
void setIndexPath(String indexPath) {
this.indexPath = indexPath;
}
/**
* @return the schemaVersion
*/
@ -60,13 +49,6 @@ class Index {
return schemaVersion;
}
/**
* @param schemaVersion the schemaVersion to set
*/
void setSchemaVersion(String schemaVersion) {
this.schemaVersion = schemaVersion;
}
/**
* @return the solrVersion
*/
@ -75,19 +57,16 @@ class Index {
}
/**
* @param solrVersion the solrVersion to set
* @return the newIndex
*/
void setSolrVersion(String solrVersion) {
this.solrVersion = solrVersion;
boolean isNewIndex() {
return newIndex;
}
/**
* @param true if all Index fields are set, false otherwise
* @param newIndex the newIndex to set
*/
boolean isIndexDataPopulated() {
if (!this.indexPath.isEmpty() && !this.solrVersion.isEmpty() && !this.schemaVersion.isEmpty()) {
return true;
}
return false;
void setNewIndex(boolean newIndex) {
this.newIndex = newIndex;
}
}

4
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java
View File

@ -73,7 +73,7 @@ class IndexFinder {
return index;
}
}
return new Index();
return null;
}
static Index createLatestVersionIndexDir(Case theCase) {
@ -91,7 +91,7 @@ class IndexFinder {
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr6_schema_1.8\index
X:\Case\ingest4\ModuleOutput\keywordsearch\data\solr7_schema_2.0\index
*/
Index bestCandidateIndex = new Index();
Index bestCandidateIndex = null;
double solrVerFound = 0.0;
double schemaVerFound = 0.0;
for (Index index : allIndexes) {

218
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -36,9 +36,11 @@ import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
@ -63,6 +65,7 @@ import org.openide.modules.Places;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.Case.CaseType;
import org.sleuthkit.autopsy.casemodule.CaseMetadata;
import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
@ -188,7 +191,6 @@ public class Server {
private int currentSolrServerPort = 0;
private int currentSolrStopPort = 0;
private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT);
private UNCPathUtilities uncPathUtilities = null;
private static final String SOLR = "solr";
private static final String CORE_PROPERTIES = "core.properties";
@ -257,7 +259,6 @@ public class Server {
}
}
currentCoreLock = new ReentrantReadWriteLock(true);
uncPathUtilities = new UNCPathUtilities();
logger.log(Level.INFO, "Created Server instance using Java at {0}", javaHome); //NON-NLS
}
@ -667,6 +668,14 @@ public class Server {
currentCoreLock.writeLock().lock();
try {
currentCore = openCore(theCase, index);
try {
// execute a test query. if it fails, an exception will be thrown
queryNumIndexedFiles();
} catch (NoOpenCoreException ex) {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
}
serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
} finally {
currentCoreLock.writeLock().unlock();
@ -739,8 +748,142 @@ public class Server {
throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex);
}
String coreName = theCase.getTextIndexName();
return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, index, theCase.getCaseType());
try {
String coreName = getCoreName(index, theCase);
File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
if (!dataDir.exists()) {
dataDir.mkdirs();
}
if (!this.isRunning()) {
logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
}
if (!coreIsLoaded(coreName)) {
/*
* The core either does not exist or it is not loaded. Make a
* request that will cause the core to be created if it does not
* exist or loaded if it already exists.
*/
// In single user mode, if there is a core.properties file already,
// we've hit a solr bug. Compensate by deleting it.
if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
Path corePropertiesFile = Paths.get(solrFolder.toString(), SOLR, coreName, CORE_PROPERTIES);
if (corePropertiesFile.toFile().exists()) {
try {
corePropertiesFile.toFile().delete();
} catch (Exception ex) {
logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
}
}
}
CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
createCoreRequest.setDataDir(dataDir.getAbsolutePath());
createCoreRequest.setCoreName(coreName);
createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
createCoreRequest.setIsLoadOnStartup(false);
createCoreRequest.setIsTransient(true);
currentSolrServer.request(createCoreRequest);
}
if (!coreIndexFolderExists(coreName)) {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
}
return new Core(coreName, theCase.getCaseType(), index);
} catch (SolrServerException | SolrException | IOException | CaseMetadata.CaseMetadataException ex) {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
}
}
/**
* Get or create a sanitized Solr core name. Stores the core name if needed.
*
* @param index Index object
* @param theCase Case object
*
* @return The sanitized Solr core name
*/
private String getCoreName(Index index, Case theCase) throws CaseMetadata.CaseMetadataException {
String coreName = "";
if (index.isNewIndex()) {
// come up with a new core name
coreName = createCoreName(theCase.getName());
// store the new core name
theCase.setTextIndexName(coreName);
} else {
// get core name
coreName = theCase.getTextIndexName();
if (coreName.isEmpty()) {
// come up with a new core name
coreName = createCoreName(theCase.getName());
// store the new core name
theCase.setTextIndexName(coreName);
}
}
return coreName;
}
/**
* Create and sanitize a core name.
*
* @param caseName Case name
*
* @return The sanitized Solr core name
*/
private String createCoreName(String caseName) {
if (caseName.isEmpty()) {
caseName = DEFAULT_CORE_NAME;
}
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
Date date = new Date();
String coreName = caseName + "_" + dateFormat.format(date);
return sanitizeCoreName(coreName);
}
/**
* Sanitizes the case name for Solr cores.
*
* Solr:
* http://stackoverflow.com/questions/29977519/what-makes-an-invalid-core-name
* may not be / \ :
* Starting Solr6: core names must consist entirely of periods, underscores, hyphens, and alphanumerics as well not start with a hyphen. may not contain space characters.
*
* @param coreName A candidate core name.
*
* @return The sanitized core name.
*/
static private String sanitizeCoreName(String coreName) {
String result;
// Remove all non-ASCII characters
result = coreName.replaceAll("[^\\p{ASCII}]", "_"); //NON-NLS
// Remove all control characters
result = result.replaceAll("[\\p{Cntrl}]", "_"); //NON-NLS
// Remove spaces / \ : ? ' "
result = result.replaceAll("[ /?:'\"\\\\]", "_"); //NON-NLS
// Make it all lowercase
result = result.toLowerCase();
// Must not start with hyphen
if (result.length() > 0 && !(Character.isLetter(result.codePointAt(0))) && !(result.codePointAt(0) == '-')) {
result = "_" + result;
}
if (result.isEmpty()) {
result = DEFAULT_CORE_NAME;
}
return result;
}
/**
@ -1083,73 +1226,6 @@ public class Server {
return Long.toString(parentID) + Server.CHUNK_ID_SEPARATOR + Integer.toString(childID);
}
/**
* Creates/opens a Solr core (index) for a case.
*
* @param coreName The core name.
* @param index The text index object for the core.
* @param caseType The type of the case (single-user or multi-user) for
* which the core is being created/opened.
*
* @return An object representing the created/opened core.
*
* @throws KeywordSearchModuleException If an error occurs while
* creating/opening the core.
*/
private Core openCore(String coreName, Index index, CaseType caseType) throws KeywordSearchModuleException {
try {
File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
if (!dataDir.exists()) {
dataDir.mkdirs();
}
if (!this.isRunning()) {
logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
}
if (!coreIsLoaded(coreName)) {
/*
* The core either does not exist or it is not loaded. Make a
* request that will cause the core to be created if it does not
* exist or loaded if it already exists.
*/
// In single user mode, if there is a core.properties file already,
// we've hit a solr bug. Compensate by deleting it.
if (caseType == CaseType.SINGLE_USER_CASE) {
Path corePropertiesFile = Paths.get(solrFolder.toString(), SOLR, coreName, CORE_PROPERTIES);
if (corePropertiesFile.toFile().exists()) {
try {
corePropertiesFile.toFile().delete();
} catch (Exception ex) {
logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
}
}
}
CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
createCoreRequest.setDataDir(dataDir.getAbsolutePath());
createCoreRequest.setCoreName(coreName);
createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
createCoreRequest.setIsLoadOnStartup(false);
createCoreRequest.setIsTransient(true);
currentSolrServer.request(createCoreRequest);
}
if (!coreIndexFolderExists(coreName)) {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
}
return new Core(coreName, caseType, index);
} catch (SolrServerException | SolrException | IOException ex) {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
}
}
/**
* Attempts to connect to the given Solr server.
*

19
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -52,7 +52,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS
private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS
private static final int IS_REACHABLE_TIMEOUT_MS = 1000;
private static final String SERVICE_NAME = "Solr Keyword Search Service";
ArtifactTextExtractor extractor = new ArtifactTextExtractor();
@ -165,14 +164,14 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
if (indexes.isEmpty()) {
// new case that doesn't have an existing index. create new index folder
currentVersionIndex = IndexFinder.createLatestVersionIndexDir(context.getCase());
currentVersionIndex.setNewIndex(true);
} else {
// check if one of the existing indexes is for latest Solr version and schema
currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes);
if (!currentVersionIndex.isIndexDataPopulated()) {
if (currentVersionIndex == null) {
// found existing index(es) but none were for latest Solr version and schema version
Index indexToUpgrade = IndexFinder.identifyIndexToUpgrade(indexes);
if (!indexToUpgrade.isIndexDataPopulated()) {
if (indexToUpgrade == null) {
// unable to find index that can be upgraded
throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
}
@ -181,7 +180,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
double indexSolrVersion = NumberUtils.toDouble(indexToUpgrade.getSolrVersion());
if (indexSolrVersion > currentSolrVersion) {
// oops!
throw new AutopsyServiceException("Unable to find index that can be upgraded to the latest version of Solr");
throw new AutopsyServiceException("Unable to find index to use for Case open");
}
else if (indexSolrVersion == currentSolrVersion) {
// latest Solr version but not latest schema. index should be used in read-only mode and not be upgraded.
@ -220,9 +219,8 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
indexUpgrader.performIndexUpgrade(indexToUpgrade, context.getCase().getTempDirectory());
// set the upgraded index as the index to be used for this case
currentVersionIndex.setIndexPath(newIndexDir);
currentVersionIndex.setSolrVersion(IndexFinder.getCurrentSolrVersion());
currentVersionIndex.setSchemaVersion(indexToUpgrade.getSchemaVersion());
currentVersionIndex = new Index(newIndexDir, IndexFinder.getCurrentSolrVersion(), indexToUpgrade.getSchemaVersion());
currentVersionIndex.setNewIndex(true);
}
}
}
@ -236,9 +234,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
MessageNotifyUtil.Notify.error(NbBundle.getMessage(KeywordSearch.class, "KeywordSearch.openCore.notification.msg"), ex.getMessage());
}
}
// ELTODO execute a test query
// ELTODO if failed, close the upgraded index?
}
/**
@ -273,6 +268,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
@Override
public String getServiceName() {
return SERVICE_NAME;
return NbBundle.getMessage(this.getClass(), "SolrSearchService.ServiceName");
}
}