diff --git a/KeywordSearch/build.xml b/KeywordSearch/build.xml index 3e4314aec7..dfd91b6f07 100644 --- a/KeywordSearch/build.xml +++ b/KeywordSearch/build.xml @@ -2,7 +2,41 @@ - + Builds, tests, and runs the project org.sleuthkit.autopsy.keywordsearch. + + + + + + + + + + diff --git a/KeywordSearch/ivy.xml b/KeywordSearch/ivy.xml new file mode 100644 index 0000000000..ced1e81c67 --- /dev/null +++ b/KeywordSearch/ivy.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/KeywordSearch/manifest.mf b/KeywordSearch/manifest.mf index cc92feb905..2a5aa19d6b 100644 --- a/KeywordSearch/manifest.mf +++ b/KeywordSearch/manifest.mf @@ -1,6 +1,7 @@ Manifest-Version: 1.0 OpenIDE-Module: org.sleuthkit.autopsy.keywordsearch/0 OpenIDE-Module-Implementation-Version: 1 +OpenIDE-Module-Install: org/sleuthkit/autopsy/keywordsearch/Installer.class OpenIDE-Module-Layer: org/sleuthkit/autopsy/keywordsearch/layer.xml OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/keywordsearch/Bundle.properties diff --git a/KeywordSearch/nbproject/genfiles.properties b/KeywordSearch/nbproject/genfiles.properties index 02e215031a..77a13dda50 100644 --- a/KeywordSearch/nbproject/genfiles.properties +++ b/KeywordSearch/nbproject/genfiles.properties @@ -3,6 +3,6 @@ build.xml.script.CRC32=87b97b04 build.xml.stylesheet.CRC32=a56c6a5b@1.46.2 # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. -nbproject/build-impl.xml.data.CRC32=eaa84b46 +nbproject/build-impl.xml.data.CRC32=ecf316f0 nbproject/build-impl.xml.script.CRC32=fe1f48d2 nbproject/build-impl.xml.stylesheet.CRC32=238281d1@1.46.2 diff --git a/KeywordSearch/nbproject/project.properties b/KeywordSearch/nbproject/project.properties index 18e9c3a275..63041f6f7a 100644 --- a/KeywordSearch/nbproject/project.properties +++ b/KeywordSearch/nbproject/project.properties @@ -1,4 +1,12 @@ -file.reference.apache-solr-solrj-3.4.0.jar=release/modules/ext/apache-solr-solrj-3.4.0.jar +file.reference.commons-codec-1.5.jar=release/modules/ext/commons-codec-1.5.jar +file.reference.commons-httpclient-3.1.jar=release/modules/ext/commons-httpclient-3.1.jar +file.reference.commons-io-1.4.jar=release/modules/ext/commons-io-1.4.jar +file.reference.jcl-over-slf4j-1.6.1.jar=release/modules/ext/jcl-over-slf4j-1.6.1.jar +file.reference.slf4j-api-1.6.1.jar=release/modules/ext/slf4j-api-1.6.1.jar +file.reference.slf4j-jdk14-1.6.1.jar=release/modules/ext/slf4j-jdk14-1.6.1.jar +file.reference.solr-solrj-3.5.0.jar=release/modules/ext/solr-solrj-3.5.0.jar javac.source=1.6 javac.compilerargs=-Xlint -Xlint:-serial +javadoc.reference.solr-solrj-3.5.0.jar=release/modules/ext/solr-solrj-3.5.0-javadoc.jar +source.reference.solr-solrj-3.5.0.jar=release/modules/ext/solr-solrj-3.5.0-sources.jar spec.version.base=0.0 diff --git a/KeywordSearch/nbproject/project.xml b/KeywordSearch/nbproject/project.xml index cd9043e97e..f82e7afeeb 100644 --- a/KeywordSearch/nbproject/project.xml +++ b/KeywordSearch/nbproject/project.xml @@ -14,6 +14,14 @@ 7.31.1 + + org.openide.modules + + + + 7.23.1 + + org.openide.nodes @@ -99,24 +107,24 @@ release/modules/ext/commons-httpclient-3.1.jar - ext/apache-solr-solrj-3.4.0.jar - release/modules/ext/apache-solr-solrj-3.4.0.jar + ext/commons-codec-1.5.jar + release/modules/ext/commons-codec-1.5.jar - ext/commons-codec-1.4.jar - release/modules/ext/commons-codec-1.4.jar - - - ext/wstx-asl-3.2.7.jar - release/modules/ext/wstx-asl-3.2.7.jar + ext/commons-lang-2.4.jar + release/modules/ext/commons-lang-2.4.jar ext/jcl-over-slf4j-1.6.1.jar release/modules/ext/jcl-over-slf4j-1.6.1.jar - ext/geronimo-stax-api_1.0_spec-1.0.1.jar - release/modules/ext/geronimo-stax-api_1.0_spec-1.0.1.jar + ext/slf4j-jdk14-1.6.1.jar + release/modules/ext/slf4j-jdk14-1.6.1.jar + + + ext/solr-solrj-3.5.0.jar + release/modules/ext/solr-solrj-3.5.0.jar diff --git a/KeywordSearch/release/modules/ext/apache-solr-solrj-3.4.0.jar b/KeywordSearch/release/modules/ext/apache-solr-solrj-3.4.0.jar deleted file mode 100644 index f615d94370..0000000000 Binary files a/KeywordSearch/release/modules/ext/apache-solr-solrj-3.4.0.jar and /dev/null differ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java index 839f3910bc..b398a081de 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractedContentViewer.java @@ -26,12 +26,13 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.openide.nodes.Node; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.corecomponentinterfaces.DataContentViewer; import org.sleuthkit.autopsy.datamodel.ContentNode; +import org.apache.commons.lang.StringEscapeUtils; + @ServiceProvider(service = DataContentViewer.class) public class ExtractedContentViewer implements DataContentViewer { @@ -62,8 +63,8 @@ public class ExtractedContentViewer implements DataContentViewer { @Override public String getMarkup() { try { - String content = getSolrContent(selectedNode); - return "
" + content + "
"; + String content = StringEscapeUtils.escapeHtml(getSolrContent(selectedNode)); + return "
" + content.trim() + "
"; } catch (SolrServerException ex) { logger.log(Level.WARNING, "Couldn't get extracted content.", ex); return ""; @@ -115,14 +116,14 @@ public class ExtractedContentViewer implements DataContentViewer { return true; } - SolrServer solr = Server.getServer().getSolr(); + Server.Core solrCore = KeywordSearch.getServer().getCore(); SolrQuery q = new SolrQuery(); q.setQuery("*:*"); q.addFilterQuery("id:" + node.getContent().getId()); q.setFields("id"); try { - return !solr.query(q).getResults().isEmpty(); + return !solrCore.query(q).getResults().isEmpty(); } catch (SolrServerException ex) { logger.log(Level.WARNING, "Couldn't determine whether content is supported.", ex); return false; @@ -136,14 +137,15 @@ public class ExtractedContentViewer implements DataContentViewer { } private String getSolrContent(ContentNode cNode) throws SolrServerException { - SolrServer solr = Server.getServer().getSolr(); + Server.Core solrCore = KeywordSearch.getServer().getCore(); SolrQuery q = new SolrQuery(); q.setQuery("*:*"); q.addFilterQuery("id:" + cNode.getContent().getId()); q.setFields("content"); + //TODO: for debugging, remove String queryURL = q.toString(); - String content = (String) solr.query(q).getResults().get(0).getFieldValue("content"); + String content = (String) solrCore.query(q).getResults().get(0).getFieldValue("content"); return content; } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java index 0b096b9035..faed12ab6a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java @@ -23,6 +23,7 @@ import java.util.logging.Logger; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.QueryResponse; +import org.sleuthkit.autopsy.keywordsearch.Server.Core; import org.sleuthkit.datamodel.Content; class HighlightedMatchesSource implements MarkupSource { @@ -30,11 +31,19 @@ class HighlightedMatchesSource implements MarkupSource { private static final Logger logger = Logger.getLogger(HighlightedMatchesSource.class.getName()); Content content; String solrQuery; - + Core solrCore; + HighlightedMatchesSource(Content content, String solrQuery) { + this(content, solrQuery, KeywordSearch.getServer().getCore()); + } + + HighlightedMatchesSource(Content content, String solrQuery, Core solrCore) { this.content = content; this.solrQuery = solrQuery; + this.solrCore = solrCore; } + + @Override public String getMarkup() { @@ -46,15 +55,19 @@ class HighlightedMatchesSource implements MarkupSource { q.setHighlightSimplePre(""); q.setHighlightSimplePost(""); q.setHighlightFragsize(0); // don't fragment the highlight + + + //TODO: remove (only for debugging) + String queryString = q.toString(); try { - QueryResponse response = Server.getServer().getSolr().query(q); + QueryResponse response = solrCore.query(q); List contentHighlights = response.getHighlighting().get(Long.toString(content.getId())).get("content"); if (contentHighlights == null) { return "No matches in content."; } else { - return "
" + contentHighlights.get(0) + "
"; + return "
" + contentHighlights.get(0).trim() + "
"; } } catch (SolrServerException ex) { throw new RuntimeException(ex); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexContentFilesAction.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexContentFilesAction.java index e5ff41d1d1..2c78fbf791 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexContentFilesAction.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexContentFilesAction.java @@ -34,22 +34,28 @@ import javax.swing.AbstractAction; import javax.swing.JDialog; import javax.swing.JFrame; import javax.swing.JOptionPane; -import javax.swing.SwingUtilities; import javax.swing.SwingWorker; import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.FsContent; public class IndexContentFilesAction extends AbstractAction { - + + private static final Logger logger = Logger.getLogger(IndexContentFilesAction.class.getName()); + private Content c; private String name; - private static final Logger logger = Logger.getLogger(IndexContentFilesAction.class.getName()); - + private Server.Core solrCore; + public IndexContentFilesAction(Content c, String name) { + this(c, name, KeywordSearch.getServer().getCore()); + } + + IndexContentFilesAction(Content c, String name, Server.Core solrCore) { super("Index files..."); this.c = c; this.name = name; + this.solrCore = solrCore; } @Override @@ -69,7 +75,7 @@ public class IndexContentFilesAction extends AbstractAction { @Override protected Integer doInBackground() throws Exception { - Ingester ingester = new Ingester("http://localhost:8983/solr"); + Ingester ingester = solrCore.getIngester(); Collection files = c.accept(new GetIngestableFilesContentVisitor()); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index fdd97a699a..b7502ebb38 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -21,14 +21,12 @@ package org.sleuthkit.autopsy.keywordsearch; import java.io.IOException; import java.io.InputStream; import java.io.Reader; -import java.net.MalformedURLException; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Logger; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; import org.apache.solr.common.SolrException; @@ -42,23 +40,11 @@ import org.sleuthkit.datamodel.FsContent; class Ingester { private static final Logger logger = Logger.getLogger(Ingester.class.getName()); - private SolrServer solr; + private SolrServer solrCore; private boolean uncommitedIngests = false; - /** - * New Ingester connected to the server at given url - * @param url Should be something like "http://localhost:8983/solr" - */ - Ingester(String url) { - try { - this.solr = new CommonsHttpSolrServer(url); - } catch (MalformedURLException ex) { - throw new RuntimeException(ex); - } - } - - Ingester(SolrServer solr) { - this.solr = solr; + Ingester(SolrServer solrCore) { + this.solrCore = solrCore; } @Override @@ -67,7 +53,7 @@ class Ingester { // Warn if files might have been left uncommited. if (uncommitedIngests) { - logger.warning("Ingester was used to add files that it never committed!"); + logger.warning("Ingester was used to add files that it never committed."); } } @@ -97,7 +83,7 @@ class Ingester { up.setParam("commit", "false"); try { - solr.request(up); + solrCore.request(up); // should't get any checked exceptions, } catch (IOException ex) { // It's possible that we will have IO errors @@ -126,7 +112,7 @@ class Ingester { void commit() { uncommitedIngests = false; try { - solr.commit(); + solrCore.commit(); // if commit doesn't work, something's broken } catch (IOException ex) { throw new RuntimeException(ex); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java new file mode 100644 index 0000000000..8bb8b39128 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java @@ -0,0 +1,35 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import org.openide.modules.ModuleInstall; +import org.sleuthkit.autopsy.casemodule.Case; + +public class Installer extends ModuleInstall{ + + @Override + public void restored() { + Case.addPropertyChangeListener(new KeywordSearch.CaseChangeListener()); + } + + // TODO: need logic for starting & shutting down the server. + // startup should be robust enough to deal with a still running server + // from a previous crash. + +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java new file mode 100644 index 0000000000..41746e1515 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearch.java @@ -0,0 +1,61 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import org.sleuthkit.autopsy.casemodule.Case; + +class KeywordSearch { + + private static final String BASE_URL = "http://localhost:8983/solr/"; + private static final Server SERVER = new Server(BASE_URL); + + static Server getServer() { + return SERVER; + } + + // don't instantiate + private KeywordSearch() { + throw new AssertionError(); + } + + static class CaseChangeListener implements PropertyChangeListener { + + CaseChangeListener() { + } + + @Override + public void propertyChange(PropertyChangeEvent evt) { + String changed = evt.getPropertyName(); + Object oldValue = evt.getOldValue(); + Object newValue = evt.getNewValue(); + + if (changed.equals(Case.CASE_CURRENT_CASE)) { + if (newValue != null) { + // new case is open + SERVER.openCore(); + } else if (oldValue != null) { + // a case was closed + SERVER.closeCore(); + } + } + } + } +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchDataExplorer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchDataExplorer.java index 8570f01e4f..b5bf4f7dd9 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchDataExplorer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchDataExplorer.java @@ -26,7 +26,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -75,7 +74,7 @@ public class KeywordSearchDataExplorer implements DataExplorer { - SolrServer solr = Server.getServer().getSolr(); + Server.Core solrCore = KeywordSearch.getServer().getCore(); SolrQuery q = new SolrQuery(); q.setQuery(solrQuery); @@ -84,12 +83,10 @@ public class KeywordSearchDataExplorer implements DataExplorer { for (int start = 0; !allMatchesFetched; start = start + ROWS_PER_FETCH) { - - q.setStart(start); try { - QueryResponse response = solr.query(q); + QueryResponse response = solrCore.query(q); SolrDocumentList resultList = response.getResults(); long results = resultList.getNumFound(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 6b30998aa8..b3a90fce72 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -18,19 +18,25 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.io.File; +import java.io.IOException; import java.net.MalformedURLException; +import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.sleuthkit.autopsy.casemodule.Case; class Server { - private static final String url = "http://localhost:8983/solr"; - private static final Server S = new Server(url); + private static final String DEFAULT_CORE_NAME = "coreCase"; + // TODO: DEFAULT_CORE_NAME needs to be replaced with unique names to support multiple open cases + - static Server getServer() { - return S; - } - private SolrServer solr; + private CommonsHttpSolrServer solr; + private String instanceDir = "C:/Users/pmartel/solr-test/maytag/solr"; Server(String url) { try { @@ -40,11 +46,105 @@ class Server { } } - Ingester getIngester() { - return new Ingester(this.solr); + void start() { } - SolrServer getSolr() { - return this.solr; + void stop() { + } + + + + /**** Convenience methods for use while we only open one case at a time ****/ + + private Core currentCore = null; + + void openCore() { + if (currentCore != null) { + throw new RuntimeException("Already an open Core!"); + } + currentCore = openCore(Case.getCurrentCase()); + } + + void closeCore() { + if (currentCore == null) { + throw new RuntimeException("No currently open Core!"); + } + currentCore.close(); + currentCore = null; + } + + Core getCore() { + if (currentCore == null) { + throw new RuntimeException("No currently open Core!"); + } + return currentCore; + } + + + /**** end single-case specific methods ****/ + + + + Core openCore(Case c) { + String sep = File.separator; + String dataDir = c.getCaseDirectory() + sep + "keywordsearch" + sep + "data"; + return this.openCore(DEFAULT_CORE_NAME, new File(dataDir)); + } + + Core openCore(String coreName, File dataDir) { + try { + if (!dataDir.exists()) { + dataDir.mkdirs(); + } + + CoreAdminRequest.Create createCore = new CoreAdminRequest.Create(); + createCore.setDataDir(dataDir.getAbsolutePath()); + createCore.setInstanceDir(instanceDir); + createCore.setCoreName(coreName); + + this.solr.request(createCore); + + return new Core(coreName); + + } catch (SolrServerException ex) { + throw new RuntimeException(ex); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + class Core { + + private String name; + // server to access a core needs to be built from a URL with the + // core in it, and is only good for core-specific operations + private SolrServer solrCore; + + private Core(String name) { + this.name = name; + try { + this.solrCore = new CommonsHttpSolrServer(solr.getBaseURL() + "/" + name); + } catch (MalformedURLException ex) { + throw new RuntimeException(ex); + } + } + + Ingester getIngester() { + return new Ingester(this.solrCore); + } + + QueryResponse query(SolrQuery sq) throws SolrServerException { + return solrCore.query(sq); + } + + void close () { + try { + CoreAdminRequest.unloadCore(this.name, solr); + } catch (SolrServerException ex) { + throw new RuntimeException(ex); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } } }