Add KeywordSearch module and Ingester class

.gitignore

@@ -1,4 +1,5 @@
 /Case/build/
+/Case/nbproject/private/
 /CoreComponentInterfaces/build/
 /DirectoryTree/build/
 /CoreComponents/build/
@@ -15,3 +16,4 @@
 /DataModel/release/modules/lib/libewf.dll
 /DataModel/release/modules/lib/libtsk_jni.dll
 /DataModel/release/modules/lib/zlib1.dll
+/KeywordSearch/build/

KeywordSearch/build.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- You may freely edit this file. See harness/README in the NetBeans platform -->
+<!-- for some information on what you could do (e.g. targets to override). -->
+<!-- If you delete this file and reopen the project it will be recreated. -->
+<project name="org.sleuthkit.autopsy.keywordsearch" default="netbeans" basedir=".">
+    <description>Builds, tests, and runs the project org.sleuthkit.autopsy.keywordsearch.</description>
+    <import file="nbproject/build-impl.xml"/>
+</project>

KeywordSearch/manifest.mf

@@ -0,0 +1,6 @@
+Manifest-Version: 1.0
+OpenIDE-Module: org.sleuthkit.autopsy.keywordsearch/0
+OpenIDE-Module-Implementation-Version: 1
+OpenIDE-Module-Layer: org/sleuthkit/autopsy/keywordsearch/layer.xml
+OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/keywordsearch/Bundle.properties
+

KeywordSearch/nbproject/build-impl.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+*** GENERATED FROM project.xml - DO NOT EDIT  ***
+***         EDIT ../build.xml INSTEAD         ***
+-->
+<project name="org.sleuthkit.autopsy.keywordsearch-impl" basedir="..">
+    <fail message="Please build using Ant 1.7.1 or higher.">
+        <condition>
+            <not>
+                <antversion atleast="1.7.1"/>
+            </not>
+        </condition>
+    </fail>
+    <property file="nbproject/private/suite-private.properties"/>
+    <property file="nbproject/suite.properties"/>
+    <fail unless="suite.dir">You must set 'suite.dir' to point to your containing module suite</fail>
+    <property file="${suite.dir}/nbproject/private/platform-private.properties"/>
+    <property file="${suite.dir}/nbproject/platform.properties"/>
+    <macrodef name="property" uri="http://www.netbeans.org/ns/nb-module-project/2">
+        <attribute name="name"/>
+        <attribute name="value"/>
+        <sequential>
+            <property name="@{name}" value="${@{value}}"/>
+        </sequential>
+    </macrodef>
+    <macrodef name="evalprops" uri="http://www.netbeans.org/ns/nb-module-project/2">
+        <attribute name="property"/>
+        <attribute name="value"/>
+        <sequential>
+            <property name="@{property}" value="@{value}"/>
+        </sequential>
+    </macrodef>
+    <property file="${user.properties.file}"/>
+    <nbmproject2:property name="harness.dir" value="nbplatform.${nbplatform.active}.harness.dir" xmlns:nbmproject2="http://www.netbeans.org/ns/nb-module-project/2"/>
+    <nbmproject2:property name="nbplatform.active.dir" value="nbplatform.${nbplatform.active}.netbeans.dest.dir" xmlns:nbmproject2="http://www.netbeans.org/ns/nb-module-project/2"/>
+    <nbmproject2:evalprops property="cluster.path.evaluated" value="${cluster.path}" xmlns:nbmproject2="http://www.netbeans.org/ns/nb-module-project/2"/>
+    <fail message="Path to 'platform' cluster missing in $${cluster.path} property or using corrupt Netbeans Platform (missing harness).">
+        <condition>
+            <not>
+                <contains string="${cluster.path.evaluated}" substring="platform"/>
+            </not>
+        </condition>
+    </fail>
+    <import file="${harness.dir}/build.xml"/>
+</project>

KeywordSearch/nbproject/genfiles.properties

@@ -0,0 +1,8 @@
+build.xml.data.CRC32=71d4527d
+build.xml.script.CRC32=87b97b04
+build.xml.stylesheet.CRC32=a56c6a5b@1.46.2
+# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
+# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
+nbproject/build-impl.xml.data.CRC32=71d4527d
+nbproject/build-impl.xml.script.CRC32=fe1f48d2
+nbproject/build-impl.xml.stylesheet.CRC32=238281d1@1.46.2

KeywordSearch/nbproject/project.properties

@@ -0,0 +1,3 @@
+javac.source=1.6
+javac.compilerargs=-Xlint -Xlint:-serial
+spec.version.base=0.0

KeywordSearch/nbproject/project.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://www.netbeans.org/ns/project/1">
+    <type>org.netbeans.modules.apisupport.project</type>
+    <configuration>
+        <data xmlns="http://www.netbeans.org/ns/nb-module-project/3">
+            <code-name-base>org.sleuthkit.autopsy.keywordsearch</code-name-base>
+            <suite-component/>
+            <module-dependencies>
+                <dependency>
+                    <code-name-base>org.sleuthkit.autopsy.datamodel</code-name-base>
+                    <build-prerequisite/>
+                    <compile-dependency/>
+                    <run-dependency>
+                        <release-version>1</release-version>
+                        <specification-version>1.0</specification-version>
+                    </run-dependency>
+                </dependency>
+            </module-dependencies>
+            <public-packages/>
+            <class-path-extension>
+                <runtime-relative-path>ext/apache-solr-solrj-3.4.0.jar</runtime-relative-path>
+                <binary-origin>release/modules/ext/apache-solr-solrj-3.4.0.jar</binary-origin>
+            </class-path-extension>
+        </data>
+    </configuration>
+</project>

KeywordSearch/nbproject/suite.properties

@@ -0,0 +1 @@
+suite.dir=${basedir}/..

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties

@@ -0,0 +1 @@
+OpenIDE-Module-Name=KeywordSearch

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java

@@ -0,0 +1,139 @@
+package org.sleuthkit.autopsy.keywordsearch;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.net.MalformedURLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.logging.Logger;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
+import org.apache.solr.common.util.ContentStream;
+import org.sleuthkit.datamodel.File;
+import org.sleuthkit.datamodel.TskException;
+
+/**
+ * Handles ingesting files to a Solr server, given the url string for it
+ */
+class Ingester {
+    
+
+    private SolrServer solr;
+    private boolean uncommitedIngests = false;
+    
+    /**
+     * New Ingester connected to the server at given url
+     * @param url Should be something like "http://localhost:8983/solr"
+     */
+    Ingester(String url) {
+        try {
+            this.solr = new CommonsHttpSolrServer(url);
+        } catch (MalformedURLException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+    
+    @Override
+    protected void finalize() throws Throwable {
+        super.finalize();
+        if (uncommitedIngests) {
+            Logger.getLogger(Ingester.class.getName()).warning("Ingester was used to add files that it never committed!");
+        }
+    }
+
+    void ingest(File f) throws IngesterException {
+        Map<String, String> fields = new HashMap<String, String>();
+        fields.put("id", Long.toString(f.getId()));
+        fields.put("file_name", f.getName());
+        fields.put("ctime", f.getCtimeAsDate());
+        fields.put("atime", f.getAtimeAsDate());
+        fields.put("mtime", f.getMtimeAsDate());
+        fields.put("crtime", f.getMtimeAsDate());
+
+        ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update/extract");
+        up.addContentStream(new FileContentStream(f));
+        setFields(up, fields);
+        up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+        
+        try {
+            solr.request(up);
+        } catch (IOException ex) {
+            throw new IngesterException("Problem posting file contents to Solr", ex);
+        } catch (SolrServerException ex) {
+            throw new IngesterException("Problem posting file contents to Solr", ex);
+        }
+        uncommitedIngests = true;
+    }
+    
+    void commit() throws IngesterException {
+        uncommitedIngests = false;
+        try {
+            solr.commit();
+        } catch (IOException ex) {
+            throw new IngesterException("Problem making Solr commit", ex);
+        } catch (SolrServerException ex) {
+            throw new IngesterException("Problem making Solr commit", ex);
+        }
+    }
+
+    private static void setFields(ContentStreamUpdateRequest up, Map<String, String> fields) {
+        for (Entry<String, String> field : fields.entrySet()) {
+            up.setParam("literal." + field.getKey(), field.getValue());
+        }
+    }
+
+    private static class FileContentStream implements ContentStream {
+
+        File f;
+
+        FileContentStream(File f) {
+            this.f = f;
+        }
+
+        @Override
+        public String getName() {
+            return f.getName();
+        }
+
+        @Override
+        public String getSourceInfo() {
+            return "File:" + f.getId();
+        }
+
+        @Override
+        public String getContentType() {
+            return null;
+        }
+
+        @Override
+        public Long getSize() {
+            return f.getSize();
+        }
+
+        @Override
+        public InputStream getStream() throws IOException {
+            try {
+                return new ByteArrayInputStream(f.read(0, f.getSize()));
+            } catch (TskException ex) {
+                throw new IOException(ex);
+            }
+        }
+
+        @Override
+        public Reader getReader() throws IOException {
+            throw new UnsupportedOperationException("Not supported yet.");
+        }
+    }
+    
+    static class IngesterException extends Exception {
+        IngesterException(String message, Throwable ex) {
+            super(message, ex);
+        }
+    }
+}

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/layer.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE filesystem PUBLIC "-//NetBeans//DTD Filesystem 1.2//EN" "http://www.netbeans.org/dtds/filesystem-1_2.dtd">
+<filesystem/>

nbproject/project.properties

@@ -17,12 +17,14 @@ modules=\
     ${project.org.sleuthkit.autopsy.filesearch}:\
     ${project.org.sleuthkit.autopsy.datamodel}:\
     ${project.org.sleuthkit.autopsy.logging}:\
-    ${project.org.sleuthkit.autopsy.casemodule}
+    ${project.org.sleuthkit.autopsy.casemodule}:\
+    ${project.org.sleuthkit.autopsy.keywordsearch}
 project.org.sleuthkit.autopsy.casemodule=Case
 project.org.sleuthkit.autopsy.corecomponentinterfaces=CoreComponentInterfaces
 project.org.sleuthkit.autopsy.corecomponents=CoreComponents
 project.org.sleuthkit.autopsy.directorytree=DirectoryTree
 project.org.sleuthkit.autopsy.filesearch=FileSearch
+project.org.sleuthkit.autopsy.keywordsearch=KeywordSearch
 project.org.sleuthkit.autopsy.logging=Logging
 project.org.sleuthkit.autopsy.menuactions=MenuActions
 project.org.sleuthkit.autopsy.datamodel=DataModel