Basic search is working, but still minor GUI bugs

This commit is contained in:
Peter J. Martel 2011-11-18 19:06:13 -05:00
parent 48154da58b
commit 0d600f4980
20 changed files with 1016 additions and 22 deletions

View File

@ -1,8 +1,8 @@
build.xml.data.CRC32=db477856
build.xml.data.CRC32=f0161c51
build.xml.script.CRC32=6ec7becb
build.xml.stylesheet.CRC32=a56c6a5b@1.42.2
build.xml.stylesheet.CRC32=a56c6a5b@1.46.2
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=db477856
nbproject/build-impl.xml.data.CRC32=f0161c51
nbproject/build-impl.xml.script.CRC32=8c5007a7
nbproject/build-impl.xml.stylesheet.CRC32=238281d1@1.42.2
nbproject/build-impl.xml.stylesheet.CRC32=238281d1@1.46.2

View File

@ -115,6 +115,15 @@
<specification-version>0.0</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.keywordsearch</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<release-version>0-1</release-version>
<specification-version>0.0</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.logging</code-name-base>
<build-prerequisite/>

View File

@ -23,7 +23,10 @@ import java.util.List;
import javax.swing.Action;
import org.openide.nodes.FilterNode;
import org.openide.nodes.Node;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.Volume;
/**
* This class sets the actions for the nodes in the directory tree and creates

View File

@ -26,6 +26,7 @@
<file name="org-sleuthkit-autopsy-directorytree-DirectoryTreeTopComponent.instance">
<attr name="instanceOf" stringvalue="org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer"/>
<attr name="instanceCreate" methodvalue="org.sleuthkit.autopsy.directorytree.DirectoryTreeTopComponent.getDefault"/>
<attr name="position" intvalue="100"/>
</file>
</folder>
<folder name="Windows2">

View File

@ -40,7 +40,6 @@ import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.border.EmptyBorder;
import org.openide.nodes.Node;
import org.openide.util.NbBundle;
import org.openide.windows.TopComponent;
import org.openide.windows.WindowManager;

View File

@ -22,6 +22,7 @@
<file name="org-sleuthkit-autopsy-filesearch-FileSearchTopComponent.instance">
<attr name="instanceOf" stringvalue="org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer"/>
<attr name="instanceCreate" methodvalue="org.sleuthkit.autopsy.filesearch.FileSearchTopComponent.getDefault"/>
<attr name="position" intvalue="200"/>
</file>
</folder>
<folder name="Windows2">

View File

@ -1,8 +1,8 @@
build.xml.data.CRC32=71d4527d
build.xml.data.CRC32=eaa84b46
build.xml.script.CRC32=87b97b04
build.xml.stylesheet.CRC32=a56c6a5b@1.46.2
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=71d4527d
nbproject/build-impl.xml.data.CRC32=eaa84b46
nbproject/build-impl.xml.script.CRC32=fe1f48d2
nbproject/build-impl.xml.stylesheet.CRC32=238281d1@1.46.2

View File

@ -1,3 +1,4 @@
file.reference.apache-solr-solrj-3.4.0.jar=release/modules/ext/apache-solr-solrj-3.4.0.jar
javac.source=1.6
javac.compilerargs=-Xlint -Xlint:-serial
spec.version.base=0.0

View File

@ -6,6 +6,73 @@
<code-name-base>org.sleuthkit.autopsy.keywordsearch</code-name-base>
<suite-component/>
<module-dependencies>
<dependency>
<code-name-base>org.openide.awt</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<specification-version>7.31.1</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.openide.nodes</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<specification-version>7.21.1</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.openide.util</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<specification-version>8.15.1</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.openide.util.lookup</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<specification-version>8.8.1</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.openide.windows</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<specification-version>6.40.1</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.casemodule</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<release-version>1</release-version>
<specification-version>1.0</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.corecomponentinterfaces</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<release-version>1</release-version>
<specification-version>1.0</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.corecomponents</code-name-base>
<build-prerequisite/>
<compile-dependency/>
<run-dependency>
<release-version>1</release-version>
<specification-version>1.0</specification-version>
</run-dependency>
</dependency>
<dependency>
<code-name-base>org.sleuthkit.autopsy.datamodel</code-name-base>
<build-prerequisite/>
@ -16,11 +83,41 @@
</run-dependency>
</dependency>
</module-dependencies>
<public-packages/>
<public-packages>
<package>org.sleuthkit.autopsy.keywordsearch</package>
</public-packages>
<class-path-extension>
<runtime-relative-path>ext/slf4j-api-1.6.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/slf4j-api-1.6.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-io-1.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-io-1.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-httpclient-3.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-httpclient-3.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/apache-solr-solrj-3.4.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/apache-solr-solrj-3.4.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-codec-1.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-codec-1.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/wstx-asl-3.2.7.jar</runtime-relative-path>
<binary-origin>release/modules/ext/wstx-asl-3.2.7.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jcl-over-slf4j-1.6.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jcl-over-slf4j-1.6.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/geronimo-stax-api_1.0_spec-1.0.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/geronimo-stax-api_1.0_spec-1.0.1.jar</binary-origin>
</class-path-extension>
</data>
</configuration>
</project>

View File

@ -1 +1,5 @@
OpenIDE-Module-Name=KeywordSearch
IndexProgressPanel.statusText.text=Status text
IndexProgressPanel.cancelButton.text=Cancel
KeywordSearchTopComponent.searchButton.text=Search
KeywordSearchTopComponent.queryLabel.text=Solr query:

View File

@ -0,0 +1,121 @@
package org.sleuthkit.autopsy.keywordsearch;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskData.FileKnown;
import org.sleuthkit.datamodel.TskException;
import org.sleuthkit.datamodel.Volume;
import org.sleuthkit.datamodel.VolumeSystem;
class GetIngestableFilesContentVisitor implements ContentVisitor<Collection<FsContent>> {
private static Logger logger = Logger.getLogger(GetIngestableFilesContentVisitor.class.getName());
// supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika
private static final String[] supportedExtensions = {"tar", "jar", "zip", "bzip2",
"gz", "tgz", "doc", "xls", "ppt", "rtf", "pdf", "html", "xhtml", "txt",
"bmp", "gif", "png", "jpeg", "tiff", "mp3", "aiff", "au", "midi", "wav",
"pst", "xml", "class"};
// the full predicate of a SQLite statement to match supported extensions
private static final String extensionsLikePredicate;
static {
StringBuilder likes = new StringBuilder("0");
for (String ext : supportedExtensions) {
likes.append(" OR (name LIKE '%.");
likes.append(ext);
likes.append("')");
}
extensionsLikePredicate = likes.toString();
}
@Override
public Collection<FsContent> visit(Directory drctr) {
return getAllFromChildren(drctr);
}
private String getExtension(String fileName) {
int lastDot = fileName.lastIndexOf(".");
if (lastDot >= 0) {
return fileName.substring(lastDot + 1, fileName.length()).toLowerCase();
} else {
return "";
}
}
@Override
public Collection<FsContent> visit(File file) {
String extension = getExtension(file.getName());
if (Arrays.asList(supportedExtensions).contains(extension)) {
return Collections.singleton((FsContent)file);
} else {
return Collections.EMPTY_LIST;
}
}
@Override
public Collection<FsContent> visit(FileSystem fs) {
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
String query = "SELECT * FROM tsk_files WHERE fs_obj_id = " + fs.getId()
//+ " AND (" + extensionsLikePredicate + ")"
+ " AND (known != " + FileKnown.KNOWN.toLong() + ")";
try {
ResultSet rs = sc.runQuery(query);
return sc.resultSetToFsContents(rs);
} catch (SQLException ex) {
logger.log(Level.WARNING, "Couldn't get all files in FileSystem", ex);
return Collections.EMPTY_SET;
}
}
@Override
public Collection<FsContent> visit(Image image) {
return getAllFromChildren(image);
}
@Override
public Collection<FsContent> visit(Volume volume) {
return getAllFromChildren(volume);
}
@Override
public Collection<FsContent> visit(VolumeSystem vs) {
return getAllFromChildren(vs);
}
private Collection<FsContent> getAllFromChildren(Content c) {
Collection<FsContent> all = new ArrayList<FsContent>();
try {
for (Content child : c.getChildren()) {
all.addAll(child.accept(this));
}
} catch (TskException ex) {
logger.log(Level.SEVERE, "Error getting Content children", ex);
}
return all;
}
}

View File

@ -0,0 +1,143 @@
package org.sleuthkit.autopsy.keywordsearch;
import java.awt.Dimension;
import java.awt.Toolkit;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.AbstractAction;
import javax.swing.JDialog;
import javax.swing.JFrame;
import javax.swing.SwingWorker;
import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.FsContent;
public class IndexContentFilesAction extends AbstractAction {
private Content c;
private String name;
private static final Logger logger = Logger.getLogger(IndexContentFilesAction.class.getName());
public IndexContentFilesAction(Content c, String name) {
super("Index files...");
this.c = c;
this.name = name;
}
@Override
public void actionPerformed(ActionEvent e) {
// create the popUp window for it
String title = "Indexing files in " + name;
final JFrame frame = new JFrame(title);
final JDialog popUpWindow = new JDialog(frame, title, true); // to make the popUp Window modal
// initialize panel
final IndexProgressPanel panel = new IndexProgressPanel();
final SwingWorker task = new SwingWorker<Void, String>() {
@Override
protected Void doInBackground() throws Exception {
Ingester ingester = new Ingester("http://localhost:8983/solr");
Collection<FsContent> files = c.accept(new GetIngestableFilesContentVisitor());
setProgress(0);
int fileCount = files.size();
int finishedFiles = 0;
for (FsContent f : files) {
if (isCancelled()) {
return null;
}
this.publish("Indexing " + (finishedFiles+1) + "/" + fileCount + ": " + f.getName());
try {
ingester.ingest(f);
} catch (IngesterException ex) {
logger.log(Level.INFO, "Ingester had a problem with file '" + f.getName() + "' (id: " + f.getId() + ").", ex);
}
setProgress(++finishedFiles * 100 / fileCount);
}
ingester.commit();
return null;
}
@Override
protected void done() {
try {
if (!this.isCancelled()) {
get();
}
} catch (InterruptedException ex) {
// shouldn't be interrupted except by cancel
throw new RuntimeException(ex);
} catch (ExecutionException ex) {
logger.log(Level.SEVERE, "Fatal error during ingest.", ex);
} finally {
popUpWindow.setVisible(false);
popUpWindow.dispose();
}
}
@Override
protected void process(List<String> messages) {
if (!messages.isEmpty()) {
panel.setStatusText(messages.get(messages.size() - 1));
}
panel.setProgressBar(getProgress());
}
};
panel.addCancelButtonActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
task.cancel(true);
}
});
popUpWindow.add(panel);
popUpWindow.pack();
popUpWindow.setResizable(false);
// set the location of the popUp Window on the center of the screen
Dimension screenDimension = Toolkit.getDefaultToolkit().getScreenSize();
double w = popUpWindow.getSize().getWidth();
double h = popUpWindow.getSize().getHeight();
popUpWindow.setLocation((int) ((screenDimension.getWidth() - w) / 2), (int) ((screenDimension.getHeight() - h) / 2));
popUpWindow.addWindowListener(new WindowAdapter() {
@Override
public void windowClosing(WindowEvent e) {
// deal with being Xed out of
if (!task.isDone()) {
task.cancel(true);
}
}
});
task.execute();
// display the window
popUpWindow.setVisible(true);
}
}

View File

@ -0,0 +1,62 @@
<?xml version="1.1" encoding="UTF-8" ?>
<Form version="1.5" maxVersion="1.7" type="org.netbeans.modules.form.forminfo.JPanelFormInfo">
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
</AuxValues>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="statusText" min="-2" max="-2" attributes="0"/>
<Component id="progressBar" alignment="0" pref="420" max="32767" attributes="0"/>
<Component id="cancelButton" alignment="1" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="statusText" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="progressBar" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="cancelButton" min="-2" max="-2" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Component class="javax.swing.JProgressBar" name="progressBar">
</Component>
<Component class="javax.swing.JLabel" name="statusText">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="IndexProgressPanel.statusText.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JButton" name="cancelButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="IndexProgressPanel.cancelButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Form>

View File

@ -0,0 +1,89 @@
/*
* IndexProgressPanel.java
*
* Created on Nov 15, 2011, 5:19:59 PM
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.awt.event.ActionListener;
/**
*
* @author pmartel
*/
public class IndexProgressPanel extends javax.swing.JPanel {
/** Creates new form IndexProgressPanel */
public IndexProgressPanel() {
initComponents();
progressBar.setMinimum(0);
progressBar.setMaximum(100);
progressBar.setIndeterminate(true);
statusText.setText("Starting...");
}
/** This method is called from within the constructor to
* initialize the form.
* WARNING: Do NOT modify this code. The content of this method is
* always regenerated by the Form Editor.
*/
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
progressBar = new javax.swing.JProgressBar();
statusText = new javax.swing.JLabel();
cancelButton = new javax.swing.JButton();
statusText.setText(org.openide.util.NbBundle.getMessage(IndexProgressPanel.class, "IndexProgressPanel.statusText.text")); // NOI18N
cancelButton.setText(org.openide.util.NbBundle.getMessage(IndexProgressPanel.class, "IndexProgressPanel.cancelButton.text")); // NOI18N
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(statusText)
.addComponent(progressBar, javax.swing.GroupLayout.DEFAULT_SIZE, 420, Short.MAX_VALUE)
.addComponent(cancelButton, javax.swing.GroupLayout.Alignment.TRAILING))
.addContainerGap())
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(statusText)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(progressBar, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(cancelButton)
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
);
}// </editor-fold>//GEN-END:initComponents
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton cancelButton;
private javax.swing.JProgressBar progressBar;
private javax.swing.JLabel statusText;
// End of variables declaration//GEN-END:variables
/**
* Sets a listener for the Cancel button
* @param e The action listener
*/
void addCancelButtonActionListener(ActionListener e) {
this.cancelButton.addActionListener(e);
}
void setProgressBar(int percent) {
progressBar.setIndeterminate(false);
progressBar.setValue(percent);
}
void setStatusText(String text) {
statusText.setText(text);
}
}

View File

@ -14,14 +14,17 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.ContentStream;
import org.sleuthkit.datamodel.File;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.TskException;
/**
* Handles ingesting files to a Solr server, given the url string for it
*/
class Ingester {
private static final Logger logger = Logger.getLogger(Ingester.class.getName());
private SolrServer solr;
@ -39,15 +42,30 @@ class Ingester {
}
}
Ingester(SolrServer solr) {
this.solr = solr;
}
@Override
protected void finalize() throws Throwable {
super.finalize();
// Warn if files might have been left uncommited.
if (uncommitedIngests) {
Logger.getLogger(Ingester.class.getName()).warning("Ingester was used to add files that it never committed!");
logger.warning("Ingester was used to add files that it never committed!");
}
}
void ingest(File f) throws IngesterException {
/**
* Sends a file to Solr to have its content extracted and added to the
* index. commit() should be called once you're done ingesting files.
*
* @param f File to ingest
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException if
* there was an error processing the given file, but the Solr server is
* probably fine.
*/
void ingest(FsContent f) throws IngesterException {
Map<String, String> fields = new HashMap<String, String>();
fields.put("id", Long.toString(f.getId()));
fields.put("file_name", f.getName());
@ -57,28 +75,43 @@ class Ingester {
fields.put("crtime", f.getMtimeAsDate());
ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update/extract");
up.addContentStream(new FileContentStream(f));
up.addContentStream(new FscContentStream(f));
setFields(up, fields);
up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
up.setParam("commit", "false");
try {
solr.request(up);
// should't get any checked exceptions, but Tika problems result in
// an unchecked SolrException
} catch (IOException ex) {
throw new IngesterException("Problem posting file contents to Solr", ex);
throw new RuntimeException(ex);
} catch (SolrServerException ex) {
throw new IngesterException("Problem posting file contents to Solr", ex);
throw new RuntimeException(ex);
} catch (SolrException ex) {
ErrorCode ec = ErrorCode.getErrorCode(ex.code());
// When Tika has problems with a document, it throws a server error
// but it's okay to continue with other documents
if (ec.equals(ErrorCode.SERVER_ERROR)) {
throw new IngesterException("Problem posting file contents to Solr. SolrException error code: " + ec , ex);
} else {
throw ex;
}
}
uncommitedIngests = true;
}
void commit() throws IngesterException {
void commit() {
uncommitedIngests = false;
try {
solr.commit();
// if commit doesn't work, something's broken
} catch (IOException ex) {
throw new IngesterException("Problem making Solr commit", ex);
throw new RuntimeException(ex);
} catch (SolrServerException ex) {
throw new IngesterException("Problem making Solr commit", ex);
throw new RuntimeException(ex);
}
}
@ -88,11 +121,11 @@ class Ingester {
}
}
private static class FileContentStream implements ContentStream {
private static class FscContentStream implements ContentStream {
File f;
FsContent f;
FileContentStream(File f) {
FscContentStream(FsContent f) {
this.f = f;
}
@ -119,9 +152,15 @@ class Ingester {
@Override
public InputStream getStream() throws IOException {
try {
return new ByteArrayInputStream(f.read(0, f.getSize()));
long size = f.getSize();
if (size > 0) {
return new ByteArrayInputStream(f.read(0, f.getSize()));
} else {
// can't read files with size 0
return new ByteArrayInputStream(new byte[0]);
}
} catch (TskException ex) {
throw new IOException(ex);
throw new IOException("Error reading file '" + f.getName() + "' (id: " + f.getId() + ")", ex);
}
}

View File

@ -0,0 +1,118 @@
package org.sleuthkit.autopsy.keywordsearch;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.beans.PropertyChangeEvent;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.openide.nodes.Node;
import org.openide.util.lookup.ServiceProvider;
import org.openide.windows.TopComponent;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer;
import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.SleuthkitCase;
@ServiceProvider(service=DataExplorer.class, position=300)
public class KeywordSearchDataExplorer implements DataExplorer {
private static KeywordSearchDataExplorer theInstance;
private KeywordSearchTopComponent tc;
public KeywordSearchDataExplorer() {
this.setTheInstance();
this.tc = new KeywordSearchTopComponent();
this.tc.addSearchButtonListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
search(tc.getQueryText());
}
});
}
private synchronized void setTheInstance() {
if (theInstance == null) {
theInstance = this;
} else {
throw new RuntimeException("NOOO!!! Mulitple instances of KeywordSearchTopComponent! BAD!");
}
}
private void search(String solrQuery) {
List<FsContent> matches = new ArrayList<FsContent>();
boolean allMatchesFetched = false;
final int ROWS_PER_FETCH = 10000;
SolrServer solr = Server.getServer().getSolr();
SolrQuery q = new SolrQuery();
q.setQuery(solrQuery);
q.setRows(ROWS_PER_FETCH);
q.setFields("id");
for (int start = 0; !allMatchesFetched; start = start + ROWS_PER_FETCH) {
q.setStart(start);
try {
QueryResponse response = solr.query(q);
SolrDocumentList resultList = response.getResults();
long results = resultList.getNumFound();
allMatchesFetched = start + ROWS_PER_FETCH >= results;
for (SolrDocument resultDoc : resultList) {
long id = Long.parseLong((String) resultDoc.getFieldValue("id"));
SleuthkitCase sc = Case.getCurrentCase().getSleuthkitCase();
// TODO: has to be a better way to get files. Also, need to
// check that we actually get 1 hit for each id
ResultSet rs = sc.runQuery("select * from tsk_files where obj_id=" + id);
matches.addAll(sc.resultSetToFsContents(rs));
rs.close();
}
} catch (SolrServerException ex) {
// TODO: handle bad query strings, among other issues
} catch (SQLException ex) {
// TODO: handle error getting files from database
}
}
String pathText = "Solr query: " + solrQuery;
Node rootNode = new KeywordSearchNode(matches, pathText);
TopComponent searchResultWin = DataResultTopComponent.createInstance("Keyword search", pathText, rootNode, matches.size());
searchResultWin.requestActive(); // make it the active top component
}
@Override
public org.openide.windows.TopComponent getTopComponent() {
return this.tc;
}
@Override
public void propertyChange(PropertyChangeEvent evt) {
}
}

View File

@ -0,0 +1,84 @@
package org.sleuthkit.autopsy.keywordsearch;
import java.sql.SQLException;
import java.util.List;
import org.openide.nodes.AbstractNode;
import org.sleuthkit.autopsy.datamodel.ContentNode;
import org.sleuthkit.autopsy.datamodel.ContentNodeVisitor;
import org.sleuthkit.autopsy.datamodel.RootContentChildren;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.TskException;
class KeywordSearchNode extends AbstractNode implements ContentNode {
private String searchText;
KeywordSearchNode(List<FsContent> keys, String searchText) {
super(new RootContentChildren(keys));
this.searchText = searchText;
}
@Override
public long getID() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public Object[][] getRowValues(int rows) throws SQLException {
int totalNodes = getChildren().getNodesCount();
Object[][] objs;
int maxRows = 0;
if(totalNodes > rows){
objs = new Object[rows][];
maxRows = rows;
}
else{
objs = new Object[totalNodes][];
maxRows = totalNodes;
}
for(int i = 0; i < maxRows; i++){
PropertySet[] props = getChildren().getNodeAt(i).getPropertySets();
Property[] property = props[0].getProperties();
objs[i] = new Object[property.length];
for(int j = 0; j < property.length; j++){
try {
objs[i][j] = property[j].getValue();
} catch (Exception ex) {
objs[i][j] = "n/a";
}
}
}
return objs;
}
@Override
public byte[] read(long offset, long len) throws TskException {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public Content getContent() {
return null;
}
@Override
public String[] getDisplayPath() {
return new String[] {this.searchText};
}
@Override
public String[] getSystemPath() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public <T> T accept(ContentNodeVisitor<T> v) {
throw new UnsupportedOperationException("Not supported yet.");
}
}

View File

@ -0,0 +1,83 @@
<?xml version="1.1" encoding="UTF-8" ?>
<Form version="1.5" maxVersion="1.7" type="org.netbeans.modules.form.forminfo.JPanelFormInfo">
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
</AuxValues>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="queryLabel" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="jScrollPane1" pref="302" max="32767" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="searchButton" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="queryLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="jScrollPane1" min="-2" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
<Component id="searchButton" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="132" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Container class="javax.swing.JScrollPane" name="jScrollPane1">
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
<SubComponents>
<Component class="javax.swing.JTextArea" name="queryTextArea">
<Properties>
<Property name="columns" type="int" value="20"/>
<Property name="rows" type="int" value="5"/>
</Properties>
</Component>
</SubComponents>
</Container>
<Component class="javax.swing.JButton" name="searchButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchTopComponent.searchButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JLabel" name="queryLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchTopComponent.queryLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Form>

View File

@ -0,0 +1,96 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
/*
* KeywordSearchTopComponent.java
*
* Created on Nov 17, 2011, 6:39:26 PM
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.awt.event.ActionListener;
import org.openide.windows.TopComponent;
/**
*
* @author pmartel
*/
public class KeywordSearchTopComponent extends TopComponent {
/** Creates new form KeywordSearchTopComponent */
public KeywordSearchTopComponent() {
initComponents();
setName("Keyword Search");
putClientProperty(TopComponent.PROP_CLOSING_DISABLED, Boolean.TRUE);
}
/** This method is called from within the constructor to
* initialize the form.
* WARNING: Do NOT modify this code. The content of this method is
* always regenerated by the Form Editor.
*/
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
jScrollPane1 = new javax.swing.JScrollPane();
queryTextArea = new javax.swing.JTextArea();
searchButton = new javax.swing.JButton();
queryLabel = new javax.swing.JLabel();
queryTextArea.setColumns(20);
queryTextArea.setRows(5);
jScrollPane1.setViewportView(queryTextArea);
searchButton.setText(org.openide.util.NbBundle.getMessage(KeywordSearchTopComponent.class, "KeywordSearchTopComponent.searchButton.text")); // NOI18N
queryLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchTopComponent.class, "KeywordSearchTopComponent.queryLabel.text")); // NOI18N
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(queryLabel))
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 302, Short.MAX_VALUE))
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(searchButton)))
.addContainerGap())
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(queryLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addGap(18, 18, 18)
.addComponent(searchButton)
.addContainerGap(132, Short.MAX_VALUE))
);
}// </editor-fold>//GEN-END:initComponents
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JScrollPane jScrollPane1;
private javax.swing.JLabel queryLabel;
private javax.swing.JTextArea queryTextArea;
private javax.swing.JButton searchButton;
// End of variables declaration//GEN-END:variables
void addSearchButtonListener(ActionListener l) {
searchButton.addActionListener(l);
}
String getQueryText() {
return queryTextArea.getText();
}
}

View File

@ -0,0 +1,44 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.net.MalformedURLException;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
/**
*
* @author pmartel
*/
public class Server {
private static final String url = "http://localhost:8983/solr";
private static final Server S = new Server(url);
static Server getServer() {
return S;
}
private SolrServer solr;
Server(String url) {
try {
this.solr = new CommonsHttpSolrServer(url);
} catch (MalformedURLException ex) {
throw new RuntimeException(ex);
}
}
Ingester getIngester() {
return new Ingester(this.solr);
}
SolrServer getSolr() {
return this.solr;
}
}