From 0bb37a029e21fa17dba485ca9d38bb07ca5a3809 Mon Sep 17 00:00:00 2001 From: Eamonn Saunders Date: Thu, 9 Apr 2015 11:58:26 -0400 Subject: [PATCH] - Added support for connecting to remote Solr server. - Refactored local Solr server startup logic. --- .../sleuthkit/autopsy/casemodule/Case.java | 25 ++- .../autopsy/casemodule/CaseSchema.xsd | 3 + .../autopsy/casemodule/XMLCaseManagement.java | 39 ++++- .../autopsy/keywordsearch/Installer.java | 147 ++---------------- .../autopsy/keywordsearch/Server.java | 104 +++++++++++-- 5 files changed, 165 insertions(+), 153 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 6c41241135..fc527d9b9d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -337,16 +337,20 @@ public class Case implements SleuthkitCase.ErrorObserver { XMLCaseManagement xmlcm = new XMLCaseManagement(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); + Date date = new Date(); + String indexName = caseName + "_" + dateFormat.format(date); + String dbName = null; - // figure out the database name + + // figure out the database name and index name for text extraction if (caseType == CaseType.SINGLE_USER_CASE) { dbName = caseDir + File.separator + "autopsy.db"; //NON-NLS } else if (caseType == CaseType.MULTI_USER_CASE) { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); - dbName = caseName + "_" + dateFormat.format(new Date()); + dbName = caseName + "_" + dateFormat.format(date); } - xmlcm.create(caseDir, caseName, examiner, caseNumber, caseType, dbName); // create a new XML config file + xmlcm.create(caseDir, caseName, examiner, caseNumber, caseType, dbName, indexName); // create a new XML config file xmlcm.writeFile(); SleuthkitCase db = null; @@ -838,6 +842,19 @@ public class Case implements SleuthkitCase.ErrorObserver { } } + /** + * Get the name of the index where extracted text is stored for the case. + * + * @return Index name. + */ + public String getTextIndexName() { + if (xmlcm == null) { + return ""; + } else { + return xmlcm.getTextIndexName(); + } + } + /** * Get absolute module output directory path where modules should save their * permanent data The directory is a subdirectory of this case dir. diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/CaseSchema.xsd b/Core/src/org/sleuthkit/autopsy/casemodule/CaseSchema.xsd index 442eae823f..63c9aa49df 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/CaseSchema.xsd +++ b/Core/src/org/sleuthkit/autopsy/casemodule/CaseSchema.xsd @@ -16,6 +16,8 @@ + + @@ -99,6 +101,7 @@ + diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/XMLCaseManagement.java b/Core/src/org/sleuthkit/autopsy/casemodule/XMLCaseManagement.java index f1ad541b5e..86b9ddc770 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/XMLCaseManagement.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/XMLCaseManagement.java @@ -60,6 +60,7 @@ import org.xml.sax.SAXException; final static String SCHEMA_VERSION_NAME = "SchemaVersion"; //NON-NLS final static String AUTOPSY_CRVERSION_NAME = "AutopsyCreatedVersion"; //NON-NLS final static String AUTOPSY_MVERSION_NAME = "AutopsySavedVersion"; //NON-NLS + final static String CASE_TEXT_INDEX_NAME = "TextIndexName"; //NON-NLS // folders inside case directory final static String LOG_FOLDER_NAME = "LogFolder"; //NON-NLS final static String LOG_FOLDER_RELPATH = "Log"; //NON-NLS @@ -88,6 +89,7 @@ import org.xml.sax.SAXException; private String autopsySavedVersion; private CaseType caseType; // The type of case: local or shared private String dbName; // The name of the database + private String textIndexName; // The name of the index where extracted text is stored. // for error handling private JPanel caller; @@ -247,6 +249,34 @@ import org.xml.sax.SAXException; } /** + * Sets the text index name internally (on local variable in this class) + * + * @param textIndexName the new name for the index where extracted text + * is stored for the case. + */ + private void setTextIndexName(String textIndexName) { + this.textIndexName= textIndexName; // change this to change the xml file if needed + } + + /** + * Gets the name of the index where extracted text is stored. + * + * @return the index name + */ + public String getTextIndexName() { + if (doc == null) { + return ""; + } else { + if (getCaseElement().getElementsByTagName(CASE_TEXT_INDEX_NAME).getLength() > 0) { + Element nameElement = (Element) getCaseElement().getElementsByTagName(CASE_TEXT_INDEX_NAME).item(0); + return nameElement.getTextContent(); + } else { + return ""; /// couldn't find one, so return a blank index name + } + } + } + + /** * Sets the examiner name internally (on local variable in this class) * * @param givenExaminer the new examiner @@ -503,8 +533,9 @@ import org.xml.sax.SAXException; * @param caseNumber case number (optional), can be empty * @param dbName the name of the database. Could be a local path, could be * a Postgre db name. + * @param textIndexName The name of the index where extracted text is stored. */ - protected void create(String dirPath, String caseName, String examiner, String caseNumber, CaseType caseType, String dbName) throws CaseActionException { + protected void create(String dirPath, String caseName, String examiner, String caseNumber, CaseType caseType, String dbName, String textIndexName) throws CaseActionException { clear(); // clear the previous data // set the case Name and Directory and the parent directory @@ -514,6 +545,7 @@ import org.xml.sax.SAXException; setNumber(caseNumber); setCaseType(caseType); setDatabaseName(dbName); + setTextIndexName(textIndexName); DocumentBuilder docBuilder; DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); @@ -593,6 +625,10 @@ import org.xml.sax.SAXException; dbNameElement.appendChild(doc.createTextNode(dbName)); caseElement.appendChild(dbNameElement); + Element indexNameElement = doc.createElement(CASE_TEXT_INDEX_NAME); // ... + indexNameElement.appendChild(doc.createTextNode(textIndexName)); + caseElement.appendChild(indexNameElement); + // write more code if needed ... } @@ -761,5 +797,6 @@ import org.xml.sax.SAXException; examiner = ""; caseType = CaseType.SINGLE_USER_CASE; dbName = ""; + textIndexName = ""; } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java index 99f4f288d4..677ad0e4c4 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java @@ -38,7 +38,6 @@ import org.sleuthkit.autopsy.coreutils.Version; class Installer extends ModuleInstall { private static final Logger logger = Logger.getLogger(Installer.class.getName()); - private final static int SERVER_START_RETRIES = 5; @Override public void restored() { @@ -48,138 +47,19 @@ class Installer extends ModuleInstall { Case.addPropertyChangeListener(new KeywordSearch.CaseChangeListener()); final Server server = KeywordSearch.getServer(); - int retries = SERVER_START_RETRIES; - - //TODO revise this logic, handle other server types, move some logic to Server class try { - //check if running from previous application instance and try to shut down - logger.log(Level.INFO, "Checking if server is running"); //NON-NLS - if (server.isRunning()) { - //TODO this could hang if other type of server is running - logger.log(Level.WARNING, "Already a server running on " + server.getCurrentSolrServerPort() //NON-NLS - + " port, maybe leftover from a previous run. Trying to shut it down."); //NON-NLS - //stop gracefully - server.stop(); - logger.log(Level.INFO, "Re-checking if server is running"); //NON-NLS - if (server.isRunning()) { - int serverPort = server.getCurrentSolrServerPort(); - int serverStopPort = server.getCurrentSolrStopPort(); - logger.log(Level.SEVERE, "There's already a server running on " //NON-NLS - + serverPort + " port that can't be shutdown."); //NON-NLS - if (!Server.isPortAvailable(serverPort)) { - reportPortError(serverPort); - } else if (!Server.isPortAvailable(serverStopPort)) { - reportStopPortError(serverStopPort); - } else { - //some other reason - reportInitError(); - } - - //in this case give up - - } else { - logger.log(Level.INFO, "Old Solr server shutdown successfully."); //NON-NLS - //make sure there really isn't a hang Solr process, in case isRunning() reported false - server.killSolr(); - } - } - } catch (KeywordSearchModuleException e) { - logger.log(Level.SEVERE, "Starting server failed, will try to kill. ", e); //NON-NLS - server.killSolr(); + server.start(); + } catch (SolrServerNoPortException ex) { + logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS + if (ex.getPortNumber() == server.getCurrentSolrServerPort()) + reportPortError(ex.getPortNumber()); + else + reportStopPortError(ex.getPortNumber()); } - - - try { - //Ensure no other process is still bound to that port, even if we think solr is not running - //Try to bind to the port 4 times at 1 second intervals. - //TODO move some of this logic to Server class - for (int i = 0; i <= 3; i++) { - logger.log(Level.INFO, "Checking if port available."); //NON-NLS - if (Server.isPortAvailable(server.getCurrentSolrServerPort())) { - logger.log(Level.INFO, "Port available, trying to start server."); //NON-NLS - server.start(); - break; - } else if (i == 3) { - logger.log(Level.INFO, "No port available, done retrying."); //NON-NLS - reportPortError(server.getCurrentSolrServerPort()); - retries = 0; - break; - } else { - try { - Thread.sleep(1000); - } catch (InterruptedException iex) { - logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS - } - } - } - } catch (SolrServerNoPortException npe) { - logger.log(Level.SEVERE, "Starting server failed due to no port available. ", npe); //NON-NLS - //try to kill it - - } catch (KeywordSearchModuleException e) { - logger.log(Level.SEVERE, "Starting server failed. ", e); //NON-NLS - } - - - //retry if needed - //TODO this loop may be now redundant - while (retries-- > 0) { - try { - Thread.sleep(1000); - } catch (InterruptedException ex) { - logger.log(Level.WARNING, "Timer interrupted."); //NON-NLS - } - - try { - logger.log(Level.INFO, "Ensuring the server is running, retries remaining: " + retries); //NON-NLS - if (!server.isRunning()) { - logger.log(Level.WARNING, "Server still not running"); //NON-NLS - try { - logger.log(Level.WARNING, "Trying to start the server. "); //NON-NLS - server.start(); - } catch (SolrServerNoPortException npe) { - logger.log(Level.SEVERE, "Starting server failed due to no port available. ", npe); //NON-NLS - } - } else { - logger.log(Level.INFO, "Server appears now running. "); //NON-NLS - break; - } - } catch (KeywordSearchModuleException ex) { - logger.log(Level.SEVERE, "Starting server failed. ", ex); //NON-NLS - //retry if has retries - } - - } //end of retry while loop - - - //last check if still not running to report errors - try { - Thread.sleep(1000); - } catch (InterruptedException ex) { - logger.log(Level.WARNING, "Timer interrupted."); //NON-NLS - } - try { - logger.log(Level.INFO, "Last check if server is running. "); //NON-NLS - if (!server.isRunning()) { - logger.log(Level.SEVERE, "Server is still not running. "); //NON-NLS - //check if port is taken or some other reason - int serverPort = server.getCurrentSolrServerPort(); - int serverStopPort = server.getCurrentSolrStopPort(); - if (!Server.isPortAvailable(serverPort)) { - reportPortError(serverPort); - } else if (!Server.isPortAvailable(serverStopPort)) { - reportStopPortError(serverStopPort); - } else { - //some other reason - reportInitError(); - } - } - } catch (KeywordSearchModuleException ex) { - logger.log(Level.SEVERE, "Starting server failed. ", ex); //NON-NLS - reportInitError(); - } - - + catch (KeywordSearchModuleException ex) { + logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS + reportInitError(ex.getMessage()); + } } @Override @@ -218,13 +98,10 @@ class Installer extends ModuleInstall { }); } - private void reportInitError() { + private void reportInitError(final String msg) { WindowManager.getDefault().invokeWhenUIReady(new Runnable() { @Override public void run() { - final String msg = NbBundle.getMessage(this.getClass(), "Installer.reportInitError", KeywordSearch.getServer().getCurrentSolrServerPort(), Version.getName(), Server.PROPERTIES_CURRENT_SERVER_PORT, Server.PROPERTIES_FILE); - MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"), msg); - MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"), msg); } }); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index de6871788c..6e9dfeb90a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -61,6 +61,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.datamodel.Content; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.client.solrj.impl.XMLResponseParser; +import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; @@ -148,7 +149,6 @@ public class Server { public static final long MAX_CONTENT_SIZE = 1L * 1024 * 1024 * 1024; private static final Logger logger = Logger.getLogger(Server.class.getName()); private static final String DEFAULT_CORE_NAME = "coreCase"; //NON-NLS - // TODO: DEFAULT_CORE_NAME needs to be replaced with unique names to support multiple open cases public static final String CORE_EVT = "CORE_EVT"; //NON-NLS public static final char ID_CHUNK_SEP = '_'; private String javaPath = "java"; //NON-NLS @@ -159,11 +159,14 @@ public class Server { static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME; static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS + static final String PROPERTIES_SOLR_SERVER_HOST = "IndexingServerHost"; //NON-NLS private static final String KEY = "jjk#09s"; //NON-NLS + static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS static final int DEFAULT_SOLR_SERVER_PORT = 23232; static final int DEFAULT_SOLR_STOP_PORT = 34343; private int currentSolrServerPort = 0; private int currentSolrStopPort = 0; + private String solrServerHost; private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT); public enum CORE_EVT_STATES { @@ -185,7 +188,7 @@ public class Server { Server() { initSettings(); - this.solrUrl = "http://localhost:" + currentSolrServerPort + "/solr"; //NON-NLS + this.solrUrl = "http://" + solrServerHost + ":" + currentSolrServerPort + "/solr"; //NON-NLS this.solrServer = new HttpSolrServer(solrUrl); serverAction = new ServerAction(); solrFolder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS @@ -196,6 +199,11 @@ public class Server { } private void initSettings() { + solrServerHost = ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_SOLR_SERVER_HOST); + if (solrServerHost == null || solrServerHost.isEmpty()) { + solrServerHost = DEFAULT_SOLR_SERVER_HOST; + } + if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT)) { try { currentSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT)); @@ -328,7 +336,7 @@ public class Server { List pids = new ArrayList(); //NOTE: these needs to be in sync with process start string in start() - final String pidsQuery = "Args.4.eq=-DSTOP.KEY=" + KEY + ",Args.7.eq=start.jar"; //NON-NLS + final String pidsQuery = "Args.4.eq=-DSTOP.KEY=" + KEY + ",Args.6.eq=start.jar"; //NON-NLS long[] pidsArr = PlatformUtil.getJavaPIDs(pidsQuery); if (pidsArr != null) { @@ -358,7 +366,55 @@ public class Server { * successful. */ void start() throws KeywordSearchModuleException, SolrServerNoPortException { + + if (isSolrLocal()) { + startLocalServer(); + } + else { + try { + SolrPingResponse response = solrServer.ping(); + } + catch (SolrServerException | IOException ex) { + throw new KeywordSearchModuleException("Failed to connect to Solr server at: " + solrUrl, ex); + } + } + } + + private void startLocalServer() throws KeywordSearchModuleException, SolrServerNoPortException { + + if (isRunning()) { + // If a Solr server is running we stop it. + stop(); + } + + if (!isPortAvailable(currentSolrServerPort)){ + // There is something already listening on our port. Let's see if + // this is from an earlier run that didn't successfully shut down + // and if so kill it. + final List pids = this.getSolrPIDs(); + + // If the culprit listening on the port is not a Solr process + // we refuse to start. + if (pids.isEmpty()) { + throw new SolrServerNoPortException(currentSolrServerPort); + } + + // Ok, we've tried to stop it above but there still appears to be + // a Solr process listening on our port so we forcefully kill it. + killSolr(); + + // If either of the ports are still in use after our attempt to kill + // previously running processes we give up and throw an exception. + if (!isPortAvailable(currentSolrServerPort)) { + throw new SolrServerNoPortException(currentSolrServerPort); + } + if (!isPortAvailable(currentSolrStopPort)) { + throw new SolrServerNoPortException(currentSolrStopPort); + } + } + logger.log(Level.INFO, "Starting Solr server from: " + solrFolder.getAbsolutePath()); //NON-NLS + if (isPortAvailable(currentSolrServerPort)) { logger.log(Level.INFO, "Port [" + currentSolrServerPort + "] available, starting Solr"); //NON-NLS try { @@ -405,6 +461,7 @@ public class Server { } catch (InterruptedException ex) { logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS } + final List pids = this.getSolrPIDs(); logger.log(Level.INFO, "New Solr process PID: " + pids); //NON-NLS } catch (SecurityException ex) { @@ -416,12 +473,9 @@ public class Server { throw new KeywordSearchModuleException( NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex); } - } else { - logger.log(Level.SEVERE, "Could not start Solr server process, port [" + currentSolrServerPort + "] not available!"); //NON-NLS - throw new SolrServerNoPortException(currentSolrServerPort); - } + } } - + /** * Checks to see if a specific port is available. * @@ -451,6 +505,14 @@ public class Server { return false; } + /** + * + * @return true if Solr is running on the local machine, false otherwise. + */ + private boolean isSolrLocal() { + return solrServerHost.equalsIgnoreCase("localhost"); + } + /** * Changes the current solr server port. Only call this after available. * @@ -477,6 +539,11 @@ public class Server { * Waits for the stop command to finish before returning. */ synchronized void stop() { + + // For a remote Solr server this is a no-op. + if (!isSolrLocal()) + return; + try { logger.log(Level.INFO, "Stopping Solr server from: " + solrFolder.getAbsolutePath()); //NON-NLS //try graceful shutdown @@ -525,11 +592,19 @@ public class Server { */ synchronized boolean isRunning() throws KeywordSearchModuleException { try { + + if (isSolrLocal()) { + if (isPortAvailable(currentSolrServerPort)) { + return false; + } + + if (curSolrProcess != null && !curSolrProcess.isAlive()) { + return false; + } + } // making a status request here instead of just doing solrServer.ping(), because // that doesn't work when there are no cores - //TODO check if port avail and return false if it is - //TODO handle timeout in cases when some other type of server on that port CoreAdminRequest.getStatus(null, solrServer); @@ -642,7 +717,8 @@ public class Server { */ private synchronized Core openCore(Case theCase) throws KeywordSearchModuleException { String dataDir = getIndexDirPath(theCase); - return this.openCore(DEFAULT_CORE_NAME, new File(dataDir)); + String coreName = theCase.getTextIndexName(); + return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir)); } /** @@ -929,7 +1005,9 @@ public class Server { CoreAdminRequest.Create createCore = new CoreAdminRequest.Create(); createCore.setDataDir(dataDir.getAbsolutePath()); - createCore.setInstanceDir(instanceDir); + if (isSolrLocal()) { + createCore.setInstanceDir(instanceDir); + } createCore.setCoreName(coreName); createCore.setConfigSet("AutopsyConfig"); @@ -939,7 +1017,7 @@ public class Server { return newCore; - } catch (SolrServerException ex) { + } catch (SolrServerException | SolrException ex) { throw new KeywordSearchModuleException( NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex); } catch (IOException ex) {