From 2e8fdb81f54ae8e6921cadf1b74f9d2080be775c Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 1 Feb 2017 16:49:20 -0500 Subject: [PATCH] First cut at saving Solr core metadata in SolrCore.properties file --- .../autopsy/keywordsearch/Index.java | 77 +++++++- .../autopsy/keywordsearch/IndexMetadata.java | 169 ++++++++++++++++++ .../autopsy/keywordsearch/Server.java | 79 +------- .../keywordsearch/SolrSearchService.java | 9 + 4 files changed, 254 insertions(+), 80 deletions(-) create mode 100644 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java index fd96a567cb..c1d1489ee1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java @@ -18,6 +18,9 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.text.SimpleDateFormat; +import java.util.Date; + /** * This class encapsulates KWS index data. */ @@ -26,13 +29,76 @@ class Index { private final String indexPath; private final String schemaVersion; private final String solrVersion; + private final String indexName; + private static final String DEFAULT_CORE_NAME = "coreCase"; //NON-NLS - Index(String indexPath, String solrVersion, String schemaVersion) { + Index(String indexPath, String solrVersion, String schemaVersion, String coreName, String caseName) { this.indexPath = indexPath; this.solrVersion = solrVersion; this.schemaVersion = schemaVersion; - } + if (coreName == null || coreName.isEmpty()) { + // come up with a new core name + coreName = createCoreName(caseName); + } + this.indexName = coreName; + } + + /** + * Create and sanitize a core name. + * + * @param caseName Case name + * + * @return The sanitized Solr core name + */ + private String createCoreName(String caseName) { + if (caseName.isEmpty()) { + caseName = DEFAULT_CORE_NAME; + } + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); + Date date = new Date(); + String coreName = caseName + "_" + dateFormat.format(date); + return sanitizeCoreName(coreName); + } + + /** + * Sanitizes the case name for Solr cores. + * + * Solr: + * http://stackoverflow.com/questions/29977519/what-makes-an-invalid-core-name + * may not be / \ : + * Starting Solr6: core names must consist entirely of periods, underscores, hyphens, and alphanumerics as well not start with a hyphen. may not contain space characters. + * + * @param coreName A candidate core name. + * + * @return The sanitized core name. + */ + static private String sanitizeCoreName(String coreName) { + String result; + + // Remove all non-ASCII characters + result = coreName.replaceAll("[^\\p{ASCII}]", "_"); //NON-NLS + + // Remove all control characters + result = result.replaceAll("[\\p{Cntrl}]", "_"); //NON-NLS + + // Remove spaces / \ : ? ' " + result = result.replaceAll("[ /?:'\"\\\\]", "_"); //NON-NLS + + // Make it all lowercase + result = result.toLowerCase(); + + // Must not start with hyphen + if (result.length() > 0 && !(Character.isLetter(result.codePointAt(0))) && !(result.codePointAt(0) == '-')) { + result = "_" + result; + } + + if (result.isEmpty()) { + result = DEFAULT_CORE_NAME; + } + + return result; + } /** * @return the indexPath */ @@ -53,4 +119,11 @@ class Index { String getSolrVersion() { return solrVersion; } + + /** + * @return the indexName + */ + String getIndexName() { + return indexName; + } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java new file mode 100644 index 0000000000..18b90004c0 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java @@ -0,0 +1,169 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.io.BufferedWriter; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.StringWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Date; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Result; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.sleuthkit.autopsy.coreutils.XMLUtil; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +/** + * Provides access to the text index metadata stored in the index metadata file. + */ +public class IndexMetadata { + + private final Path metadataFilePath; + private final String metadataFileName = "SolrCore.properties"; + private final static String ROOT_ELEMENT_NAME = "SolrCores"; //NON-NLS + private final static String CORE_ELEMENT_NAME = "Core"; //NON-NLS + private final static String CORE_NAME_ELEMENT_NAME = "CoreName"; //NON-NLS + private final static String SCHEMA_VERSION_ELEMENT_NAME = "SchemaVersion"; //NON-NLS + private final static String SOLR_VERSION_ELEMENT_NAME = "SolrVersion"; //NON-NLS + private final static String TEXT_INDEX_PATH_ELEMENT_NAME = "TextIndexPath"; //NON-NLS + private String coreName; + private String textIndexPath; + private String solrVersion; + private String schemaVersion; + + IndexMetadata(String caseDirectory, Index index) throws TextIndexMetadataException { + metadataFilePath = Paths.get(caseDirectory, metadataFileName); + this.coreName = index.getIndexName(); + this.solrVersion = index.getSolrVersion(); + this.schemaVersion = index.getSchemaVersion(); + this.textIndexPath = index.getIndexPath(); + writeToFile(); + } + + + + /** + * Writes the case metadata to the metadata file. + * + * @throws CaseMetadataException If there is an error writing to the case + * metadata file. + */ + private void writeToFile() throws TextIndexMetadataException { + try { + /* + * Create the XML DOM. + */ + Document doc = XMLUtil.createDocument(); + createXMLDOM(doc); + doc.normalize(); + + /* + * Prepare the DOM for pretty printing to the metadata file. + */ + Source source = new DOMSource(doc); + StringWriter stringWriter = new StringWriter(); + Result streamResult = new StreamResult(stringWriter); + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); //NON-NLS + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); //NON-NLS + transformer.transform(source, streamResult); + + /* + * Write the DOM to the metadata file. + */ + try (BufferedWriter fileWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(metadataFilePath.toFile())))) { + fileWriter.write(stringWriter.toString()); + fileWriter.flush(); + } + + } catch (ParserConfigurationException | TransformerException | IOException ex) { + throw new TextIndexMetadataException(String.format("Error writing to case metadata file %s", metadataFilePath), ex); + } + } + + /* + * Creates an XML DOM from the case metadata. + */ + private void createXMLDOM(Document doc) { + /* + * Create the root element and its children. + */ + Element rootElement = doc.createElement(ROOT_ELEMENT_NAME); + doc.appendChild(rootElement); + createChildElement(doc, rootElement, CORE_NAME_ELEMENT_NAME, createdDate); + createChildElement(doc, rootElement, SOLR_VERSION_ELEMENT_NAME, DATE_FORMAT.format(new Date())); + createChildElement(doc, rootElement, SCHEMA_VERSION_ELEMENT_NAME, CURRENT_SCHEMA_VERSION); + createChildElement(doc, rootElement, TEXT_INDEX_PATH_ELEMENT_NAME, createdByVersion); + Element caseElement = doc.createElement(CORE_ELEMENT_NAME); + rootElement.appendChild(caseElement); + + /* + * Create the children of the case element. + */ + createChildElement(doc, caseElement, CASE_NAME_ELEMENT_NAME, caseName); + createChildElement(doc, caseElement, CASE_DISPLAY_NAME_ELEMENT_NAME, caseDisplayName); + createChildElement(doc, caseElement, CASE_NUMBER_ELEMENT_NAME, caseNumber); + createChildElement(doc, caseElement, EXAMINER_ELEMENT_NAME, examiner); + createChildElement(doc, caseElement, CASE_TYPE_ELEMENT_NAME, caseType.toString()); + createChildElement(doc, caseElement, CASE_DATABASE_ELEMENT_NAME, caseDatabaseName); + createChildElement(doc, caseElement, TEXT_INDEX_ELEMENT, textIndexName); + } + + /** + * Creates an XML element for the case metadata XML DOM. + * + * @param doc The document. + * @param parentElement The parent element of the element to be created. + * @param elementName The name of the element to be created. + * @param elementContent The text content of the element to be created, may + * be empty. + */ + private void createChildElement(Document doc, Element parentElement, String elementName, String elementContent) { + Element element = doc.createElement(elementName); + element.appendChild(doc.createTextNode(elementContent)); + parentElement.appendChild(element); + } + + /** + * Exception thrown by the IndexMetadata class when there is a problem + * accessing the metadata for a text index. + */ + public final static class TextIndexMetadataException extends Exception { + + private static final long serialVersionUID = 1L; + + private TextIndexMetadataException(String message) { + super(message); + } + + private TextIndexMetadataException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 3787ef7da0..319fb09fe2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -174,7 +174,6 @@ public class Server { //max content size we can send to Solr public static final long MAX_CONTENT_SIZE = 1L * 31 * 1024 * 1024; private static final Logger logger = Logger.getLogger(Server.class.getName()); - private static final String DEFAULT_CORE_NAME = "coreCase"; //NON-NLS public static final String CORE_EVT = "CORE_EVT"; //NON-NLS @Deprecated public static final char ID_CHUNK_SEP = '_'; @@ -816,7 +815,7 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg")); } - String coreName = getCoreName(theCase); + String coreName = index.getIndexName(); if (!coreIsLoaded(coreName)) { /* * The core either does not exist or it is not loaded. Make a @@ -856,82 +855,6 @@ public class Server { throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex); } } - - /** - * Get or create a sanitized Solr core name. Stores the core name if needed. - * - * @param theCase Case object - * - * @return The sanitized Solr core name - */ - private String getCoreName(Case theCase) throws CaseMetadata.CaseMetadataException { - // get core name - String coreName = theCase.getTextIndexName(); - if (coreName == null || coreName.isEmpty()) { - // come up with a new core name - coreName = createCoreName(theCase.getName()); - // store the new core name - theCase.setTextIndexName(coreName); - } - return coreName; - } - - /** - * Create and sanitize a core name. - * - * @param caseName Case name - * - * @return The sanitized Solr core name - */ - private String createCoreName(String caseName) { - if (caseName.isEmpty()) { - caseName = DEFAULT_CORE_NAME; - } - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); - Date date = new Date(); - String coreName = caseName + "_" + dateFormat.format(date); - return sanitizeCoreName(coreName); - } - - /** - * Sanitizes the case name for Solr cores. - * - * Solr: - * http://stackoverflow.com/questions/29977519/what-makes-an-invalid-core-name - * may not be / \ : - * Starting Solr6: core names must consist entirely of periods, underscores, hyphens, and alphanumerics as well not start with a hyphen. may not contain space characters. - * - * @param coreName A candidate core name. - * - * @return The sanitized core name. - */ - static private String sanitizeCoreName(String coreName) { - - String result; - - // Remove all non-ASCII characters - result = coreName.replaceAll("[^\\p{ASCII}]", "_"); //NON-NLS - - // Remove all control characters - result = result.replaceAll("[\\p{Cntrl}]", "_"); //NON-NLS - - // Remove spaces / \ : ? ' " - result = result.replaceAll("[ /?:'\"\\\\]", "_"); //NON-NLS - - // Make it all lowercase - result = result.toLowerCase(); - - // Must not start with hyphen - if (result.length() > 0 && !(Character.isLetter(result.codePointAt(0))) && !(result.codePointAt(0) == '-')) { - result = "_" + result; - } - - if (result.isEmpty()) { - result = DEFAULT_CORE_NAME; - } - - return result; - } /** * Commits current core if it exists diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 941590bec2..d81a76c433 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -32,9 +32,11 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.math.NumberUtils; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; +import org.sleuthkit.autopsy.casemodule.CaseMetadata; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.framework.AutopsyService; @@ -300,6 +302,13 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex); } catch (KeywordSearchModuleException ex) { throw new AutopsyServiceException(String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex); + } + + try { + // store the new core name + context.getCase().setTextIndexName(currentVersionIndex.getIndexName()); + } catch (CaseMetadata.CaseMetadataException ex) { + throw new AutopsyServiceException("Failed to save core name in case metadata file", ex); } progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);