mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-16 09:47:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

First cut at saving Solr core metadata in SolrCore.properties file

Browse Source
This commit is contained in:
Eugene Livis 2017-02-01 16:49:20 -05:00
parent 3a546b7a4e
commit 2e8fdb81f5
4 changed files with 254 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java
View File

@ -18,6 +18,9 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* This class encapsulates KWS index data.
*/
@ -26,13 +29,76 @@ class Index {
private final String indexPath;
private final String schemaVersion;
private final String solrVersion;
private final String indexName;
private static final String DEFAULT_CORE_NAME = "coreCase"; //NON-NLS
Index(String indexPath, String solrVersion, String schemaVersion) {
Index(String indexPath, String solrVersion, String schemaVersion, String coreName, String caseName) {
this.indexPath = indexPath;
this.solrVersion = solrVersion;
this.schemaVersion = schemaVersion;
}
if (coreName == null || coreName.isEmpty()) {
// come up with a new core name
coreName = createCoreName(caseName);
}
this.indexName = coreName;
}
/**
* Create and sanitize a core name.
*
* @param caseName Case name
*
* @return The sanitized Solr core name
*/
private String createCoreName(String caseName) {
if (caseName.isEmpty()) {
caseName = DEFAULT_CORE_NAME;
}
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
Date date = new Date();
String coreName = caseName + "_" + dateFormat.format(date);
return sanitizeCoreName(coreName);
}
/**
* Sanitizes the case name for Solr cores.
*
* Solr:
* http://stackoverflow.com/questions/29977519/what-makes-an-invalid-core-name
* may not be / \ :
* Starting Solr6: core names must consist entirely of periods, underscores, hyphens, and alphanumerics as well not start with a hyphen. may not contain space characters.
*
* @param coreName A candidate core name.
*
* @return The sanitized core name.
*/
static private String sanitizeCoreName(String coreName) {
String result;
// Remove all non-ASCII characters
result = coreName.replaceAll("[^\\p{ASCII}]", "_"); //NON-NLS
// Remove all control characters
result = result.replaceAll("[\\p{Cntrl}]", "_"); //NON-NLS
// Remove spaces / \ : ? ' "
result = result.replaceAll("[ /?:'\"\\\\]", "_"); //NON-NLS
// Make it all lowercase
result = result.toLowerCase();
// Must not start with hyphen
if (result.length() > 0 && !(Character.isLetter(result.codePointAt(0))) && !(result.codePointAt(0) == '-')) {
result = "_" + result;
}
if (result.isEmpty()) {
result = DEFAULT_CORE_NAME;
}
return result;
}
/**
* @return the indexPath
*/
@ -53,4 +119,11 @@ class Index {
String getSolrVersion() {
return solrVersion;
}
/**
* @return the indexName
*/
String getIndexName() {
return indexName;
}
}

169
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.sleuthkit.autopsy.coreutils.XMLUtil;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
/**
* Provides access to the text index metadata stored in the index metadata file.
*/
public class IndexMetadata {
private final Path metadataFilePath;
private final String metadataFileName = "SolrCore.properties";
private final static String ROOT_ELEMENT_NAME = "SolrCores"; //NON-NLS
private final static String CORE_ELEMENT_NAME = "Core"; //NON-NLS
private final static String CORE_NAME_ELEMENT_NAME = "CoreName"; //NON-NLS
private final static String SCHEMA_VERSION_ELEMENT_NAME = "SchemaVersion"; //NON-NLS
private final static String SOLR_VERSION_ELEMENT_NAME = "SolrVersion"; //NON-NLS
private final static String TEXT_INDEX_PATH_ELEMENT_NAME = "TextIndexPath"; //NON-NLS
private String coreName;
private String textIndexPath;
private String solrVersion;
private String schemaVersion;
IndexMetadata(String caseDirectory, Index index) throws TextIndexMetadataException {
metadataFilePath = Paths.get(caseDirectory, metadataFileName);
this.coreName = index.getIndexName();
this.solrVersion = index.getSolrVersion();
this.schemaVersion = index.getSchemaVersion();
this.textIndexPath = index.getIndexPath();
writeToFile();
}
/**
* Writes the case metadata to the metadata file.
*
* @throws CaseMetadataException If there is an error writing to the case
* metadata file.
*/
private void writeToFile() throws TextIndexMetadataException {
try {
/*
* Create the XML DOM.
*/
Document doc = XMLUtil.createDocument();
createXMLDOM(doc);
doc.normalize();
/*
* Prepare the DOM for pretty printing to the metadata file.
*/
Source source = new DOMSource(doc);
StringWriter stringWriter = new StringWriter();
Result streamResult = new StreamResult(stringWriter);
Transformer transformer = TransformerFactory.newInstance().newTransformer();
transformer.setOutputProperty(OutputKeys.INDENT, "yes"); //NON-NLS
transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); //NON-NLS
transformer.transform(source, streamResult);
/*
* Write the DOM to the metadata file.
*/
try (BufferedWriter fileWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(metadataFilePath.toFile())))) {
fileWriter.write(stringWriter.toString());
fileWriter.flush();
}
} catch (ParserConfigurationException | TransformerException | IOException ex) {
throw new TextIndexMetadataException(String.format("Error writing to case metadata file %s", metadataFilePath), ex);
}
}
/*
* Creates an XML DOM from the case metadata.
*/
private void createXMLDOM(Document doc) {
/*
* Create the root element and its children.
*/
Element rootElement = doc.createElement(ROOT_ELEMENT_NAME);
doc.appendChild(rootElement);
createChildElement(doc, rootElement, CORE_NAME_ELEMENT_NAME, createdDate);
createChildElement(doc, rootElement, SOLR_VERSION_ELEMENT_NAME, DATE_FORMAT.format(new Date()));
createChildElement(doc, rootElement, SCHEMA_VERSION_ELEMENT_NAME, CURRENT_SCHEMA_VERSION);
createChildElement(doc, rootElement, TEXT_INDEX_PATH_ELEMENT_NAME, createdByVersion);
Element caseElement = doc.createElement(CORE_ELEMENT_NAME);
rootElement.appendChild(caseElement);
/*
* Create the children of the case element.
*/
createChildElement(doc, caseElement, CASE_NAME_ELEMENT_NAME, caseName);
createChildElement(doc, caseElement, CASE_DISPLAY_NAME_ELEMENT_NAME, caseDisplayName);
createChildElement(doc, caseElement, CASE_NUMBER_ELEMENT_NAME, caseNumber);
createChildElement(doc, caseElement, EXAMINER_ELEMENT_NAME, examiner);
createChildElement(doc, caseElement, CASE_TYPE_ELEMENT_NAME, caseType.toString());
createChildElement(doc, caseElement, CASE_DATABASE_ELEMENT_NAME, caseDatabaseName);
createChildElement(doc, caseElement, TEXT_INDEX_ELEMENT, textIndexName);
}
/**
* Creates an XML element for the case metadata XML DOM.
*
* @param doc The document.
* @param parentElement The parent element of the element to be created.
* @param elementName The name of the element to be created.
* @param elementContent The text content of the element to be created, may
* be empty.
*/
private void createChildElement(Document doc, Element parentElement, String elementName, String elementContent) {
Element element = doc.createElement(elementName);
element.appendChild(doc.createTextNode(elementContent));
parentElement.appendChild(element);
}
/**
* Exception thrown by the IndexMetadata class when there is a problem
* accessing the metadata for a text index.
*/
public final static class TextIndexMetadataException extends Exception {
private static final long serialVersionUID = 1L;
private TextIndexMetadataException(String message) {
super(message);
}
private TextIndexMetadataException(String message, Throwable cause) {
super(message, cause);
}
}
}

79
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -174,7 +174,6 @@ public class Server {
//max content size we can send to Solr
public static final long MAX_CONTENT_SIZE = 1L * 31 * 1024 * 1024;
private static final Logger logger = Logger.getLogger(Server.class.getName());
private static final String DEFAULT_CORE_NAME = "coreCase"; //NON-NLS
public static final String CORE_EVT = "CORE_EVT"; //NON-NLS
@Deprecated
public static final char ID_CHUNK_SEP = '_';
@ -816,7 +815,7 @@ public class Server {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
}
String coreName = getCoreName(theCase);
String coreName = index.getIndexName();
if (!coreIsLoaded(coreName)) {
/*
* The core either does not exist or it is not loaded. Make a
@ -856,82 +855,6 @@ public class Server {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
}
}
/**
* Get or create a sanitized Solr core name. Stores the core name if needed.
*
* @param theCase Case object
*
* @return The sanitized Solr core name
*/
private String getCoreName(Case theCase) throws CaseMetadata.CaseMetadataException {
// get core name
String coreName = theCase.getTextIndexName();
if (coreName == null || coreName.isEmpty()) {
// come up with a new core name
coreName = createCoreName(theCase.getName());
// store the new core name
theCase.setTextIndexName(coreName);
}
return coreName;
}
/**
* Create and sanitize a core name.
*
* @param caseName Case name
*
* @return The sanitized Solr core name
*/
private String createCoreName(String caseName) {
if (caseName.isEmpty()) {
caseName = DEFAULT_CORE_NAME;
}
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
Date date = new Date();
String coreName = caseName + "_" + dateFormat.format(date);
return sanitizeCoreName(coreName);
}
/**
* Sanitizes the case name for Solr cores.
*
* Solr:
* http://stackoverflow.com/questions/29977519/what-makes-an-invalid-core-name
* may not be / \ :
* Starting Solr6: core names must consist entirely of periods, underscores, hyphens, and alphanumerics as well not start with a hyphen. may not contain space characters.
*
* @param coreName A candidate core name.
*
* @return The sanitized core name.
*/
static private String sanitizeCoreName(String coreName) {
String result;
// Remove all non-ASCII characters
result = coreName.replaceAll("[^\\p{ASCII}]", "_"); //NON-NLS
// Remove all control characters
result = result.replaceAll("[\\p{Cntrl}]", "_"); //NON-NLS
// Remove spaces / \ : ? ' "
result = result.replaceAll("[ /?:'\"\\\\]", "_"); //NON-NLS
// Make it all lowercase
result = result.toLowerCase();
// Must not start with hyphen
if (result.length() > 0 && !(Character.isLetter(result.codePointAt(0))) && !(result.codePointAt(0) == '-')) {
result = "_" + result;
}
if (result.isEmpty()) {
result = DEFAULT_CORE_NAME;
}
return result;
}
/**
* Commits current core if it exists

9
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -32,9 +32,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.openide.util.Exceptions;
import org.openide.util.NbBundle;
import org.openide.util.lookup.ServiceProvider;
import org.openide.util.lookup.ServiceProviders;
import org.sleuthkit.autopsy.casemodule.CaseMetadata;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.framework.AutopsyService;
@ -300,6 +302,13 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
KeywordSearch.getServer().openCoreForCase(context.getCase(), currentVersionIndex);
} catch (KeywordSearchModuleException ex) {
throw new AutopsyServiceException(String.format("Failed to open or create core for %s", context.getCase().getCaseDirectory()), ex);
}
try {
// store the new core name
context.getCase().setTextIndexName(currentVersionIndex.getIndexName());
} catch (CaseMetadata.CaseMetadataException ex) {
throw new AutopsyServiceException("Failed to save core name in case metadata file", ex);
}
progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);