mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-12 07:56:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement DIY solr cloud

Browse Source
This commit is contained in:
Ann Priestman 2018-02-23 16:37:34 -05:00
parent b6b4f09909
commit 112b57776d
5 changed files with 158 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Core/src/org/sleuthkit/autopsy/casemodule/Case.java
View File

@ -806,7 +806,7 @@ public class Case {
*
* @throws CaseActionException throw if could not create the case dir
*/
static void createCaseDirectory(String caseDir, CaseType caseType) throws CaseActionException {
public static void createCaseDirectory(String caseDir, CaseType caseType) throws CaseActionException {
File caseDirF = new File(caseDir);

11
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

@ -97,6 +97,8 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestJobStartResult;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.IngestModuleError;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException;
import org.sleuthkit.autopsy.keywordsearch.Server;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.SleuthkitCase;
@ -2156,6 +2158,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
Case.openAsCurrentCase(metadataFilePath.toString());
} else {
caseDirectoryPath = PathUtils.createCaseFolderPath(rootOutputDirectory, caseName);
// Create the case directory now in case it is needed by selectSolrServerForCase
Case.createCaseDirectory(caseDirectoryPath.toString(), CaseType.MULTI_USER_CASE);
// If a list of servers exists, choose one to use for this case
Server.selectSolrServerForCase(rootOutputDirectory, caseDirectoryPath);
CaseDetails caseDetails = new CaseDetails(caseName);
Case.createAsCurrentCase(CaseType.MULTI_USER_CASE, caseDirectoryPath.toString(), caseDetails);
/*
@ -2170,6 +2179,8 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
SYS_LOGGER.log(Level.INFO, "Opened case {0} for {1}", new Object[]{caseForJob.getName(), manifest.getFilePath()});
return caseForJob;
} catch (KeywordSearchModuleException ex) {
throw new CaseManagementException(String.format("Error creating solr settings file for case %s for %s", caseName, manifest.getFilePath()), ex);
} catch (CaseActionException ex) {
throw new CaseManagementException(String.format("Error creating or opening case %s for %s", caseName, manifest.getFilePath()), ex);
} catch (IllegalStateException ex) {

5
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -184,15 +184,16 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (Case.getCurrentCase().getCaseType() == Case.CaseType.MULTI_USER_CASE) {
// for multi-user cases need to verify connection to remore SOLR server
KeywordSearchService kwsService = new SolrSearchService();
Server.IndexingServerProperties properties = Server.getMultiUserServerProperties(Case.getCurrentCase().getCaseDirectory());
int port;
try {
port = Integer.parseInt(UserPreferences.getIndexingServerPort());
port = Integer.parseInt(properties.getPort());
} catch (NumberFormatException ex) {
// if there is an error parsing the port number
throw new IngestModuleException(Bundle.KeywordSearchIngestModule_init_badInitMsg() + " " + Bundle.SolrConnectionCheck_Port(), ex);
}
try {
kwsService.tryConnect(UserPreferences.getIndexingServerHost(), port);
kwsService.tryConnect(properties.getHost(), port);
} catch (KeywordSearchServiceException ex) {
throw new IngestModuleException(Bundle.KeywordSearchIngestModule_init_badInitMsg(), ex);
}

150
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -39,7 +39,9 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Level;
import javax.swing.AbstractAction;
@ -63,6 +65,7 @@ import org.openide.modules.Places;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.Case.CaseType;
import org.sleuthkit.autopsy.casemodule.CaseMetadata;
import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
@ -721,16 +724,15 @@ public class Server {
*/
@NbBundle.Messages({
"# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
void deleteCore(String coreName, Case.CaseType caseType) throws KeywordSearchServiceException {
void deleteCore(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException {
try {
HttpSolrServer solrServer;
if (caseType == CaseType.SINGLE_USER_CASE) {
if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) {
Integer localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
solrServer = new HttpSolrServer("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
} else {
String host = UserPreferences.getIndexingServerHost();
String port = UserPreferences.getIndexingServerPort();
solrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory());
solrServer = new HttpSolrServer("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); //NON-NLS
}
connectToSolrServer(solrServer);
CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
@ -768,9 +770,8 @@ public class Server {
if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
currentSolrServer = this.localSolrServer;
} else {
String host = UserPreferences.getIndexingServerHost();
String port = UserPreferences.getIndexingServerPort();
currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
IndexingServerProperties properties = getMultiUserServerProperties(theCase.getCaseDirectory());
currentSolrServer = new HttpSolrServer("http://" + properties.getHost() + ":" + properties.getPort() + "/solr"); //NON-NLS
}
connectToSolrServer(currentSolrServer);
@ -830,6 +831,139 @@ public class Server {
}
}
/**
* Get the host and port for a multiuser case.
* If the file solrserver.txt exists, then use the values from that file.
* Otherwise use the settings from the properties file.
*
* @param caseDirectory Current case directory
* @return IndexingServerProperties containing the solr host/port for this case
*/
public static IndexingServerProperties getMultiUserServerProperties(String caseDirectory) {
Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt");
if(serverFilePath.toFile().exists()){
try{
List<String> lines = Files.readAllLines(serverFilePath);
if(lines.isEmpty()) {
logger.log(Level.SEVERE, "solrserver.txt file does not contain any data");
} else if (! lines.get(0).contains(",")) {
logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
} else {
String[] parts = lines.get(0).split(",");
if(parts.length != 2) {
logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
} else {
return new IndexingServerProperties(parts[0], parts[1]);
}
}
} catch (IOException ex) {
logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex);
}
}
// Default back to the user preferences if the solrserver.txt file was not found or if an error occurred
String host = UserPreferences.getIndexingServerHost();
String port = UserPreferences.getIndexingServerPort();
return new IndexingServerProperties(host, port);
}
/**
* Pick a solr server to use for this case and record it in the case directory.
* Looks for a file named "solrServerList.txt" in the root output directory -
* if this does not exist then no server is recorded.
*
* Format of solrServerList.txt:
* <host>,<port>
* Ex: 10.1.2.34,8983
*
* @param rootOutputDirectory
* @param caseDirectoryPath
* @throws KeywordSearchModuleException
*/
public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath) throws KeywordSearchModuleException {
// Look for the solr server list file
String serverListName = "solrServerList.txt";
Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
if(serverListPath.toFile().exists()){
// Read the list of solr servers
List<String> lines;
try{
lines = Files.readAllLines(serverListPath);
} catch (IOException ex){
throw new KeywordSearchModuleException(serverListName + " could not be read", ex);
}
// Remove any lines that don't contain a comma (these are likely just whitespace)
for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
String line = iterator.next();
if (! line.contains(",")) {
// Remove the current element from the iterator and the list.
iterator.remove();
}
}
if(lines.isEmpty()) {
throw new KeywordSearchModuleException(serverListName + " had no valid server information");
}
// Choose which server to use
int rnd = new Random().nextInt(lines.size());
String[] parts = lines.get(rnd).split(",");
if(parts.length != 2) {
throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
}
// Split it up just to do a sanity check on the data
String host = parts[0];
String port = parts[1];
if(host.isEmpty() || port.isEmpty()) {
throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
}
// Write the server data to a file
Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt");
try {
caseDirectoryPath.toFile().mkdirs();
if (! caseDirectoryPath.toFile().exists()) {
throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist");
}
Files.write(serverFile, lines.get(rnd).getBytes());
} catch (IOException ex){
throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex);
}
}
}
/**
* Helper class to store the current server properties
*/
public static class IndexingServerProperties {
private final String host;
private final String port;
IndexingServerProperties (String host, String port) {
this.host = host;
this.port = port;
}
/**
* Get the host
* @return host
*/
public String getHost() {
return host;
}
/**
* Get the port
* @return port
*/
public String getPort() {
return port;
}
}
/**
* Commits current core if it exists
*

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -173,7 +173,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
* Unload/delete the core on the server and then delete the text
* index files.
*/
KeywordSearch.getServer().deleteCore(index.getIndexName(), metadata.getCaseType());
KeywordSearch.getServer().deleteCore(index.getIndexName(), metadata);
if (!FileUtil.deleteDir(new File(index.getIndexPath()).getParentFile())) {
throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_failedToDeleteIndexFiles(index.getIndexPath()));
}