| ").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append(" | ")
.append(!examiner.isEmpty() ? examiner : NbBundle
.getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer"))
diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportKML.java b/Core/src/org/sleuthkit/autopsy/report/ReportKML.java
new file mode 100644
index 0000000000..05cbb3b6a7
--- /dev/null
+++ b/Core/src/org/sleuthkit/autopsy/report/ReportKML.java
@@ -0,0 +1,318 @@
+/*
+ *
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2014 Basis Technology Corp.
+ * contact: carrier sleuthkit org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.report;
+
+import javax.swing.JPanel;
+
+import org.openide.util.NbBundle;
+import org.sleuthkit.autopsy.casemodule.Case;
+import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.datamodel.*;
+import org.sleuthkit.autopsy.ingest.IngestManager;
+import org.sleuthkit.datamodel.BlackboardArtifact;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.logging.Level;
+import org.jdom2.Document;
+import org.jdom2.Element;
+import org.jdom2.Namespace;
+import org.jdom2.output.Format;
+import org.jdom2.output.XMLOutputter;
+import org.apache.commons.lang.StringEscapeUtils;
+
+/**
+ * Generates a KML file based on geo coordinates store in blackboard.
+ */
+class ReportKML implements GeneralReportModule {
+
+ private static final Logger logger = Logger.getLogger(ReportKML.class.getName());
+ private static ReportKML instance = null;
+ private Case currentCase;
+ private SleuthkitCase skCase;
+ private String reportPath;
+
+ // Hidden constructor for the report
+ private ReportKML() {
+ }
+
+ // Get the default implementation of this report
+ public static synchronized ReportKML getDefault() {
+ if (instance == null) {
+ instance = new ReportKML();
+ }
+ return instance;
+ }
+
+ /**
+ * Generates a body file format report for use with the MAC time tool.
+ *
+ * @param path path to save the report
+ * @param progressPanel panel to update the report's progress
+ */
+ @Override
+ public void generateReport(String path, ReportProgressPanel progressPanel) {
+
+ // Start the progress bar and setup the report
+ progressPanel.setIndeterminate(false);
+ progressPanel.start();
+ progressPanel.updateStatusLabel(NbBundle.getMessage(this.getClass(), "ReportKML.progress.querying"));
+ reportPath = path + "ReportKML.kml";
+ String reportPath2 = path + "ReportKML.txt";
+ currentCase = Case.getCurrentCase();
+ skCase = currentCase.getSleuthkitCase();
+
+ progressPanel.updateStatusLabel(NbBundle.getMessage(this.getClass(), "ReportKML.progress.loading"));
+ // Check if ingest has finished
+ String ingestwarning = "";
+ if (IngestManager.getInstance().isIngestRunning()) {
+ ingestwarning = NbBundle.getMessage(this.getClass(), "ReportBodyFile.ingestWarning.text");
+ }
+ progressPanel.setMaximumProgress(5);
+ progressPanel.increment();
+
+
+ try {
+
+ BufferedWriter out = null;
+ try {
+ out = new BufferedWriter(new FileWriter(reportPath2));
+
+ double lat = 0; // temp latitude
+ double lon = 0; //temp longitude
+ AbstractFile aFile;
+ String geoPath = ""; // will hold values of images to put in kml
+ String imageName = "";
+
+
+ File f;
+ for (BlackboardArtifact artifact : skCase.getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)) {
+ lat = 0;
+ lon = 0;
+ geoPath = "";
+ String extractedToPath;
+ for (BlackboardAttribute attribute : artifact.getAttributes()) {
+ if (attribute.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID()) //latitude
+ {
+
+ lat = attribute.getValueDouble();
+ }
+ if (attribute.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID()) //longitude
+ {
+ lon = attribute.getValueDouble();
+ }
+ }
+ if (lon != 0 && lat != 0) {
+ aFile = artifact.getSleuthkitCase().getAbstractFileById(artifact.getObjectID());
+
+ extractedToPath = reportPath + aFile.getName();
+ geoPath = extractedToPath;
+ f = new File(extractedToPath);
+ f.createNewFile();
+ copyFileUsingStream(aFile, f);
+ imageName = aFile.getName();
+ out.write(String.valueOf(lat));
+ out.write(";");
+ out.write(String.valueOf(lon));
+ out.write(";");
+ out.write(String.valueOf(geoPath));
+ out.write(";");
+ out.write(String.valueOf(imageName));
+ out.write("\n");
+ // lat lon path name
+ }
+ }
+ out.flush();
+ out.close();
+ progressPanel.increment();
+ /*
+ * Step 1: generate XML stub
+ */
+ Namespace ns = Namespace.getNamespace("", "http://earth.google.com/kml/2.2");
+ // kml
+ Element kml = new Element("kml", ns);
+ Document kmlDocument = new Document(kml);
+
+ // Document
+ Element document = new Element("Document", ns);
+ kml.addContent(document);
+
+ // name
+ Element name = new Element("name", ns);
+ name.setText("Java Generated KML Document");
+ document.addContent(name);
+
+ /*
+ * Step 2: add in Style elements
+ */
+
+ // Style
+ Element style = new Element("Style", ns);
+ style.setAttribute("id", "redIcon");
+ document.addContent(style);
+
+ // IconStyle
+ Element iconStyle = new Element("IconStyle", ns);
+ style.addContent(iconStyle);
+
+ // color
+ Element color = new Element("color", ns);
+ color.setText("990000ff");
+ iconStyle.addContent(color);
+
+ // Icon
+ Element icon = new Element("Icon", ns);
+ iconStyle.addContent(icon);
+
+ // href
+ Element href = new Element("href", ns);
+ href.setText("http://www.cs.mun.ca/~hoeber/teaching/cs4767/notes/02.1-kml/circle.png");
+ icon.addContent(href);
+ progressPanel.increment();
+ /*
+ * Step 3: read data from source location and
+ * add in a Placemark for each data element
+ */
+
+ File file = new File(reportPath2);
+ BufferedReader reader;
+
+ reader = new BufferedReader(new FileReader(file));
+
+ String line = reader.readLine();
+ while (line != null) {
+ String[] lineParts = line.split(";");
+ if (lineParts.length == 4) {
+ String coordinates = lineParts[1].trim() + "," + lineParts[0].trim(); //lat,lon
+ // Placemark
+ Element placemark = new Element("Placemark", ns);
+ document.addContent(placemark);
+
+ // name
+ Element pmName = new Element("name", ns);
+ pmName.setText(lineParts[3].trim());
+ placemark.addContent(pmName);
+
+ // Path
+ Element pmPath = new Element("Path", ns);
+ pmPath.setText(lineParts[2].trim());
+ placemark.addContent(pmPath);
+
+ // description
+ Element pmDescription = new Element("description", ns);
+ String xml = "
-
-
-
-
- Builds, tests, and runs the project org.sleuthkit.autopsy.exifextract.
-
-
\ No newline at end of file
diff --git a/ExifParser/manifest.mf b/ExifParser/manifest.mf
deleted file mode 100644
index 40668bc3a3..0000000000
--- a/ExifParser/manifest.mf
+++ /dev/null
@@ -1,6 +0,0 @@
-Manifest-Version: 1.0
-AutoUpdate-Show-In-Client: true
-OpenIDE-Module: org.sleuthkit.autopsy.exifparser/3
-OpenIDE-Module-Implementation-Version: 10
-OpenIDE-Module-Layer: org/sleuthkit/autopsy/exifparser/layer.xml
-OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/exifparser/Bundle.properties
diff --git a/ExifParser/nbproject/build-impl.xml b/ExifParser/nbproject/build-impl.xml
deleted file mode 100644
index e1bcfa276c..0000000000
--- a/ExifParser/nbproject/build-impl.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- You must set 'suite.dir' to point to your containing module suite
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/ExifParser/nbproject/project.properties b/ExifParser/nbproject/project.properties
deleted file mode 100644
index f278b6de1b..0000000000
--- a/ExifParser/nbproject/project.properties
+++ /dev/null
@@ -1,6 +0,0 @@
-javac.source=1.7
-javac.compilerargs=-Xlint -Xlint:-serial
-license.file=../LICENSE-2.0.txt
-nbm.homepage=http://www.sleuthkit.org/autopsy/
-nbm.needs.restart=true
-spec.version.base=1.1
diff --git a/ExifParser/nbproject/project.xml b/ExifParser/nbproject/project.xml
deleted file mode 100644
index 88405985e9..0000000000
--- a/ExifParser/nbproject/project.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
- org.netbeans.modules.apisupport.project
-
-
- org.sleuthkit.autopsy.exifparser
-
-
-
- org.openide.util
-
-
-
- 8.29.3
-
-
-
- org.openide.util.lookup
-
-
-
- 8.19.1
-
-
-
- org.sleuthkit.autopsy.core
-
-
-
- 9
- 7.1
-
-
-
-
-
- ext/xmpcore.jar
- release/modules/ext/xmpcore.jar
-
-
- ext/metadata-extractor-2.6.2.jar
- release/modules/ext/metadata-extractor-2.6.2.jar
-
-
-
-
diff --git a/ExifParser/nbproject/suite.properties b/ExifParser/nbproject/suite.properties
deleted file mode 100644
index 29d7cc9bd6..0000000000
--- a/ExifParser/nbproject/suite.properties
+++ /dev/null
@@ -1 +0,0 @@
-suite.dir=${basedir}/..
diff --git a/FileExtMismatch/build.xml b/FileExtMismatch/build.xml
deleted file mode 100644
index 90f199456e..0000000000
--- a/FileExtMismatch/build.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
- Builds, tests, and runs the project org.sleuthkit.autopsy.fileextmismatch.
-
-
diff --git a/FileExtMismatch/manifest.mf b/FileExtMismatch/manifest.mf
deleted file mode 100644
index c1678923ea..0000000000
--- a/FileExtMismatch/manifest.mf
+++ /dev/null
@@ -1,5 +0,0 @@
-Manifest-Version: 1.0
-OpenIDE-Module: org.sleuthkit.autopsy.fileextmismatch/3
-OpenIDE-Module-Implementation-Version: 1
-OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/fileextmismatch/Bundle.properties
-
diff --git a/FileExtMismatch/nbproject/build-impl.xml b/FileExtMismatch/nbproject/build-impl.xml
deleted file mode 100644
index 40c37cb50f..0000000000
--- a/FileExtMismatch/nbproject/build-impl.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- You must set 'suite.dir' to point to your containing module suite
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/FileExtMismatch/nbproject/platform.properties b/FileExtMismatch/nbproject/platform.properties
deleted file mode 100644
index a9fa87f749..0000000000
--- a/FileExtMismatch/nbproject/platform.properties
+++ /dev/null
@@ -1,120 +0,0 @@
-branding.token=autopsy
-netbeans-plat-version=7.3.1
-suite.dir=${basedir}
-nbplatform.active.dir=${suite.dir}/netbeans-plat/${netbeans-plat-version}
-harness.dir=${nbplatform.active.dir}/harness
-bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar
-autoupdate.catalog.url=http://dlc.sun.com.edgesuite.net/netbeans/updates/${netbeans-plat-version}/uc/final/distribution/catalog.xml.gz
-cluster.path=\
- ${nbplatform.active.dir}/harness:\
- ${nbplatform.active.dir}/java:\
- ${nbplatform.active.dir}/platform
-disabled.modules=\
- org.apache.tools.ant.module,\
- org.netbeans.api.debugger.jpda,\
- org.netbeans.api.java,\
- org.netbeans.lib.nbjavac,\
- org.netbeans.libs.cglib,\
- org.netbeans.libs.javacapi,\
- org.netbeans.libs.javacimpl,\
- org.netbeans.libs.springframework,\
- org.netbeans.modules.ant.browsetask,\
- org.netbeans.modules.ant.debugger,\
- org.netbeans.modules.ant.freeform,\
- org.netbeans.modules.ant.grammar,\
- org.netbeans.modules.ant.kit,\
- org.netbeans.modules.beans,\
- org.netbeans.modules.classfile,\
- org.netbeans.modules.dbschema,\
- org.netbeans.modules.debugger.jpda,\
- org.netbeans.modules.debugger.jpda.ant,\
- org.netbeans.modules.debugger.jpda.kit,\
- org.netbeans.modules.debugger.jpda.projects,\
- org.netbeans.modules.debugger.jpda.ui,\
- org.netbeans.modules.debugger.jpda.visual,\
- org.netbeans.modules.findbugs.installer,\
- org.netbeans.modules.form,\
- org.netbeans.modules.form.binding,\
- org.netbeans.modules.form.j2ee,\
- org.netbeans.modules.form.kit,\
- org.netbeans.modules.form.nb,\
- org.netbeans.modules.form.refactoring,\
- org.netbeans.modules.hibernate,\
- org.netbeans.modules.hibernatelib,\
- org.netbeans.modules.hudson.ant,\
- org.netbeans.modules.hudson.maven,\
- org.netbeans.modules.i18n,\
- org.netbeans.modules.i18n.form,\
- org.netbeans.modules.j2ee.core.utilities,\
- org.netbeans.modules.j2ee.eclipselink,\
- org.netbeans.modules.j2ee.eclipselinkmodelgen,\
- org.netbeans.modules.j2ee.jpa.refactoring,\
- org.netbeans.modules.j2ee.jpa.verification,\
- org.netbeans.modules.j2ee.metadata,\
- org.netbeans.modules.j2ee.metadata.model.support,\
- org.netbeans.modules.j2ee.persistence,\
- org.netbeans.modules.j2ee.persistence.kit,\
- org.netbeans.modules.j2ee.persistenceapi,\
- org.netbeans.modules.java.api.common,\
- org.netbeans.modules.java.debug,\
- org.netbeans.modules.java.editor,\
- org.netbeans.modules.java.editor.lib,\
- org.netbeans.modules.java.examples,\
- org.netbeans.modules.java.freeform,\
- org.netbeans.modules.java.guards,\
- org.netbeans.modules.java.helpset,\
- org.netbeans.modules.java.hints,\
- org.netbeans.modules.java.hints.declarative,\
- org.netbeans.modules.java.hints.declarative.test,\
- org.netbeans.modules.java.hints.legacy.spi,\
- org.netbeans.modules.java.hints.test,\
- org.netbeans.modules.java.hints.ui,\
- org.netbeans.modules.java.j2seplatform,\
- org.netbeans.modules.java.j2seproject,\
- org.netbeans.modules.java.kit,\
- org.netbeans.modules.java.lexer,\
- org.netbeans.modules.java.navigation,\
- org.netbeans.modules.java.platform,\
- org.netbeans.modules.java.preprocessorbridge,\
- org.netbeans.modules.java.project,\
- org.netbeans.modules.java.source,\
- org.netbeans.modules.java.source.ant,\
- org.netbeans.modules.java.source.queries,\
- org.netbeans.modules.java.source.queriesimpl,\
- org.netbeans.modules.java.sourceui,\
- org.netbeans.modules.java.testrunner,\
- org.netbeans.modules.javadoc,\
- org.netbeans.modules.javawebstart,\
- org.netbeans.modules.junit,\
- org.netbeans.modules.maven,\
- org.netbeans.modules.maven.checkstyle,\
- org.netbeans.modules.maven.coverage,\
- org.netbeans.modules.maven.embedder,\
- org.netbeans.modules.maven.grammar,\
- org.netbeans.modules.maven.graph,\
- org.netbeans.modules.maven.hints,\
- org.netbeans.modules.maven.indexer,\
- org.netbeans.modules.maven.junit,\
- org.netbeans.modules.maven.kit,\
- org.netbeans.modules.maven.model,\
- org.netbeans.modules.maven.osgi,\
- org.netbeans.modules.maven.persistence,\
- org.netbeans.modules.maven.refactoring,\
- org.netbeans.modules.maven.repository,\
- org.netbeans.modules.maven.search,\
- org.netbeans.modules.maven.spring,\
- org.netbeans.modules.projectimport.eclipse.core,\
- org.netbeans.modules.projectimport.eclipse.j2se,\
- org.netbeans.modules.refactoring.java,\
- org.netbeans.modules.spellchecker.bindings.java,\
- org.netbeans.modules.spring.beans,\
- org.netbeans.modules.testng,\
- org.netbeans.modules.testng.ant,\
- org.netbeans.modules.testng.maven,\
- org.netbeans.modules.websvc.jaxws21,\
- org.netbeans.modules.websvc.jaxws21api,\
- org.netbeans.modules.websvc.saas.codegen.java,\
- org.netbeans.modules.xml.jaxb,\
- org.netbeans.modules.xml.tools.java,\
- org.netbeans.spi.java.hints
-
diff --git a/FileExtMismatch/nbproject/project.properties b/FileExtMismatch/nbproject/project.properties
deleted file mode 100644
index 21d3761899..0000000000
--- a/FileExtMismatch/nbproject/project.properties
+++ /dev/null
@@ -1,3 +0,0 @@
-javac.source=1.7
-javac.compilerargs=-Xlint -Xlint:-serial
-spec.version.base=1.3
diff --git a/FileExtMismatch/nbproject/project.xml b/FileExtMismatch/nbproject/project.xml
deleted file mode 100644
index 23fa0b7529..0000000000
--- a/FileExtMismatch/nbproject/project.xml
+++ /dev/null
@@ -1,115 +0,0 @@
-
-
- org.netbeans.modules.apisupport.project
-
-
- org.sleuthkit.autopsy.fileextmismatch
-
-
-
- org.netbeans.api.progress
-
-
-
- 1
- 1.32.1
-
-
-
- org.netbeans.modules.options.api
-
-
-
- 1
- 1.31.2
-
-
-
- org.openide.awt
-
-
-
- 7.55.1
-
-
-
- org.openide.dialogs
-
-
-
- 7.28.1
-
-
-
- org.openide.explorer
-
-
-
- 6.50.3
-
-
-
- org.openide.nodes
-
-
-
- 7.33.2
-
-
-
- org.openide.util
-
-
-
- 8.29.3
-
-
-
- org.openide.util.lookup
-
-
-
- 8.19.1
-
-
-
- org.openide.windows
-
-
-
- 6.60.1
-
-
-
- org.sleuthkit.autopsy.core
-
-
-
- 9
- 7.0
-
-
-
- org.sleuthkit.autopsy.corelibs
-
-
-
- 3
- 1.1
-
-
-
- org.sleuthkit.autopsy.filetypeid
-
-
-
- 1.0
-
-
-
-
- org.sleuthkit.autopsy.fileextmismatch
-
-
-
-
diff --git a/FileExtMismatch/nbproject/suite.properties b/FileExtMismatch/nbproject/suite.properties
deleted file mode 100644
index 29d7cc9bd6..0000000000
--- a/FileExtMismatch/nbproject/suite.properties
+++ /dev/null
@@ -1 +0,0 @@
-suite.dir=${basedir}/..
diff --git a/FileTypeId/build.xml b/FileTypeId/build.xml
deleted file mode 100644
index 9771a82a0b..0000000000
--- a/FileTypeId/build.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
- Builds, tests, and runs the project org.sleuthkit.autopsy.filetypeid.
-
-
diff --git a/FileTypeId/manifest.mf b/FileTypeId/manifest.mf
deleted file mode 100644
index eac3c30420..0000000000
--- a/FileTypeId/manifest.mf
+++ /dev/null
@@ -1,5 +0,0 @@
-Manifest-Version: 1.0
-OpenIDE-Module: org.sleuthkit.autopsy.filetypeid
-OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/filetypeid/Bundle.properties
-OpenIDE-Module-Specification-Version: 1.0
-
diff --git a/FileTypeId/nbproject/build-impl.xml b/FileTypeId/nbproject/build-impl.xml
deleted file mode 100644
index ec094e8ceb..0000000000
--- a/FileTypeId/nbproject/build-impl.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- You must set 'suite.dir' to point to your containing module suite
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/FileTypeId/nbproject/platform.properties b/FileTypeId/nbproject/platform.properties
deleted file mode 100644
index a9fa87f749..0000000000
--- a/FileTypeId/nbproject/platform.properties
+++ /dev/null
@@ -1,120 +0,0 @@
-branding.token=autopsy
-netbeans-plat-version=7.3.1
-suite.dir=${basedir}
-nbplatform.active.dir=${suite.dir}/netbeans-plat/${netbeans-plat-version}
-harness.dir=${nbplatform.active.dir}/harness
-bootstrap.url=http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/netbeans/harness/tasks.jar
-autoupdate.catalog.url=http://dlc.sun.com.edgesuite.net/netbeans/updates/${netbeans-plat-version}/uc/final/distribution/catalog.xml.gz
-cluster.path=\
- ${nbplatform.active.dir}/harness:\
- ${nbplatform.active.dir}/java:\
- ${nbplatform.active.dir}/platform
-disabled.modules=\
- org.apache.tools.ant.module,\
- org.netbeans.api.debugger.jpda,\
- org.netbeans.api.java,\
- org.netbeans.lib.nbjavac,\
- org.netbeans.libs.cglib,\
- org.netbeans.libs.javacapi,\
- org.netbeans.libs.javacimpl,\
- org.netbeans.libs.springframework,\
- org.netbeans.modules.ant.browsetask,\
- org.netbeans.modules.ant.debugger,\
- org.netbeans.modules.ant.freeform,\
- org.netbeans.modules.ant.grammar,\
- org.netbeans.modules.ant.kit,\
- org.netbeans.modules.beans,\
- org.netbeans.modules.classfile,\
- org.netbeans.modules.dbschema,\
- org.netbeans.modules.debugger.jpda,\
- org.netbeans.modules.debugger.jpda.ant,\
- org.netbeans.modules.debugger.jpda.kit,\
- org.netbeans.modules.debugger.jpda.projects,\
- org.netbeans.modules.debugger.jpda.ui,\
- org.netbeans.modules.debugger.jpda.visual,\
- org.netbeans.modules.findbugs.installer,\
- org.netbeans.modules.form,\
- org.netbeans.modules.form.binding,\
- org.netbeans.modules.form.j2ee,\
- org.netbeans.modules.form.kit,\
- org.netbeans.modules.form.nb,\
- org.netbeans.modules.form.refactoring,\
- org.netbeans.modules.hibernate,\
- org.netbeans.modules.hibernatelib,\
- org.netbeans.modules.hudson.ant,\
- org.netbeans.modules.hudson.maven,\
- org.netbeans.modules.i18n,\
- org.netbeans.modules.i18n.form,\
- org.netbeans.modules.j2ee.core.utilities,\
- org.netbeans.modules.j2ee.eclipselink,\
- org.netbeans.modules.j2ee.eclipselinkmodelgen,\
- org.netbeans.modules.j2ee.jpa.refactoring,\
- org.netbeans.modules.j2ee.jpa.verification,\
- org.netbeans.modules.j2ee.metadata,\
- org.netbeans.modules.j2ee.metadata.model.support,\
- org.netbeans.modules.j2ee.persistence,\
- org.netbeans.modules.j2ee.persistence.kit,\
- org.netbeans.modules.j2ee.persistenceapi,\
- org.netbeans.modules.java.api.common,\
- org.netbeans.modules.java.debug,\
- org.netbeans.modules.java.editor,\
- org.netbeans.modules.java.editor.lib,\
- org.netbeans.modules.java.examples,\
- org.netbeans.modules.java.freeform,\
- org.netbeans.modules.java.guards,\
- org.netbeans.modules.java.helpset,\
- org.netbeans.modules.java.hints,\
- org.netbeans.modules.java.hints.declarative,\
- org.netbeans.modules.java.hints.declarative.test,\
- org.netbeans.modules.java.hints.legacy.spi,\
- org.netbeans.modules.java.hints.test,\
- org.netbeans.modules.java.hints.ui,\
- org.netbeans.modules.java.j2seplatform,\
- org.netbeans.modules.java.j2seproject,\
- org.netbeans.modules.java.kit,\
- org.netbeans.modules.java.lexer,\
- org.netbeans.modules.java.navigation,\
- org.netbeans.modules.java.platform,\
- org.netbeans.modules.java.preprocessorbridge,\
- org.netbeans.modules.java.project,\
- org.netbeans.modules.java.source,\
- org.netbeans.modules.java.source.ant,\
- org.netbeans.modules.java.source.queries,\
- org.netbeans.modules.java.source.queriesimpl,\
- org.netbeans.modules.java.sourceui,\
- org.netbeans.modules.java.testrunner,\
- org.netbeans.modules.javadoc,\
- org.netbeans.modules.javawebstart,\
- org.netbeans.modules.junit,\
- org.netbeans.modules.maven,\
- org.netbeans.modules.maven.checkstyle,\
- org.netbeans.modules.maven.coverage,\
- org.netbeans.modules.maven.embedder,\
- org.netbeans.modules.maven.grammar,\
- org.netbeans.modules.maven.graph,\
- org.netbeans.modules.maven.hints,\
- org.netbeans.modules.maven.indexer,\
- org.netbeans.modules.maven.junit,\
- org.netbeans.modules.maven.kit,\
- org.netbeans.modules.maven.model,\
- org.netbeans.modules.maven.osgi,\
- org.netbeans.modules.maven.persistence,\
- org.netbeans.modules.maven.refactoring,\
- org.netbeans.modules.maven.repository,\
- org.netbeans.modules.maven.search,\
- org.netbeans.modules.maven.spring,\
- org.netbeans.modules.projectimport.eclipse.core,\
- org.netbeans.modules.projectimport.eclipse.j2se,\
- org.netbeans.modules.refactoring.java,\
- org.netbeans.modules.spellchecker.bindings.java,\
- org.netbeans.modules.spring.beans,\
- org.netbeans.modules.testng,\
- org.netbeans.modules.testng.ant,\
- org.netbeans.modules.testng.maven,\
- org.netbeans.modules.websvc.jaxws21,\
- org.netbeans.modules.websvc.jaxws21api,\
- org.netbeans.modules.websvc.saas.codegen.java,\
- org.netbeans.modules.xml.jaxb,\
- org.netbeans.modules.xml.tools.java,\
- org.netbeans.spi.java.hints
-
diff --git a/FileTypeId/nbproject/project.properties b/FileTypeId/nbproject/project.properties
deleted file mode 100644
index 2dfc44febe..0000000000
--- a/FileTypeId/nbproject/project.properties
+++ /dev/null
@@ -1,3 +0,0 @@
-file.reference.tika-core-1.2.jar=release/modules/ext/tika-core-1.2.jar
-javac.source=1.7
-javac.compilerargs=-Xlint -Xlint:-serial
diff --git a/FileTypeId/nbproject/project.xml b/FileTypeId/nbproject/project.xml
deleted file mode 100644
index 8d3ea957d3..0000000000
--- a/FileTypeId/nbproject/project.xml
+++ /dev/null
@@ -1,103 +0,0 @@
-
-
- org.netbeans.modules.apisupport.project
-
-
- org.sleuthkit.autopsy.filetypeid
-
-
-
- org.netbeans.api.progress
-
-
-
- 1
- 1.32.1
-
-
-
- org.netbeans.modules.options.api
-
-
-
- 1
- 1.31.2
-
-
-
- org.openide.awt
-
-
-
- 7.55.1
-
-
-
- org.openide.dialogs
-
-
-
- 7.28.1
-
-
-
- org.openide.nodes
-
-
-
- 7.33.2
-
-
-
- org.openide.util
-
-
-
- 8.29.3
-
-
-
- org.openide.util.lookup
-
-
-
- 8.19.1
-
-
-
- org.openide.windows
-
-
-
- 6.60.1
-
-
-
- org.sleuthkit.autopsy.core
-
-
-
- 9
- 7.0
-
-
-
- org.sleuthkit.autopsy.corelibs
-
-
-
- 3
- 1.1
-
-
-
-
- org.sleuthkit.autopsy.filetypeid
-
-
- ext/tika-core-1.2.jar
- release/modules/ext/tika-core-1.2.jar
-
-
-
-
diff --git a/FileTypeId/nbproject/suite.properties b/FileTypeId/nbproject/suite.properties
deleted file mode 100644
index 29d7cc9bd6..0000000000
--- a/FileTypeId/nbproject/suite.properties
+++ /dev/null
@@ -1 +0,0 @@
-suite.dir=${basedir}/..
diff --git a/FileTypeId/src-alt/JMimeMagicFileTypeDetector.java b/FileTypeId/src-alt/JMimeMagicFileTypeDetector.java
deleted file mode 100644
index 246fadc81e..0000000000
--- a/FileTypeId/src-alt/JMimeMagicFileTypeDetector.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Autopsy Forensic Browser
- *
- * Copyright 2011 - 2013 Basis Technology Corp.
- * Contact: carrier sleuthkit org
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sleuthkit.autopsy.filetypeid;
-
-import org.sleuthkit.datamodel.AbstractFile;
-import net.sf.jmimemagic.Magic;
-import net.sf.jmimemagic.MagicMatch;
-import net.sf.jmimemagic.MagicMatchNotFoundException;
-import org.openide.util.Exceptions;
-
-/**
- *
- */
-public class JMimeMagicFileTypeDetector implements FileTypeDetectionInterface {
-
- @Override
- public FileIdInfo attemptMatch(AbstractFile abstractFile) {
- try {
- FileIdInfo ret = new FileIdInfo();
- final int maxBytesInitial = 3000; //how many bytes to read on first pass
- byte buffer[] = new byte[maxBytesInitial];
- ///@todo decide to use max bytes or give the whole file
- int len = abstractFile.read(buffer, 0, maxBytesInitial);
-
- try {
- MagicMatch match = Magic.getMagicMatch(buffer);
- if (match != null) {
- String matchStr = match.getMimeType();
- if (matchStr.equals("???")) {
- String desc = match.getDescription();
- if (!desc.isEmpty()) {
- ret.type = desc;
- }
- } else {
- ret.type = matchStr;
- }
- ret.extension = match.getExtension();
- }
- } catch (MagicMatchNotFoundException ex) {
- //do nothing
- }
-
- return ret;
-
- } catch (Exception ex) {
- Exceptions.printStackTrace(ex);
- return new FileIdInfo();
- }
- }
-
-}
diff --git a/FileTypeId/src-alt/MimeUtilFileTypeDetector.java b/FileTypeId/src-alt/MimeUtilFileTypeDetector.java
deleted file mode 100644
index 40d5a74b58..0000000000
--- a/FileTypeId/src-alt/MimeUtilFileTypeDetector.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Autopsy Forensic Browser
- *
- * Copyright 2011 - 2013 Basis Technology Corp.
- * Contact: carrier sleuthkit org
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.sleuthkit.autopsy.filetypeid;
-
-import eu.medsea.mimeutil.MimeException;
-import eu.medsea.mimeutil.MimeType;
-import eu.medsea.mimeutil.detector.MagicMimeMimeDetector;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import org.openide.util.Exceptions;
-import org.sleuthkit.datamodel.AbstractFile;
-
-/**
- *
- */
-public class MimeUtilFileTypeDetector implements FileTypeDetectionInterface {
- private static MagicMimeMimeDetector mimeUtil = new MagicMimeMimeDetector();
-
-
- @Override
- public FileIdInfo attemptMatch(AbstractFile abstractFile) {
- try {
- FileIdInfo ret = new FileIdInfo();
- final int maxBytesInitial = 3000; //how many bytes to read on first pass
- byte buffer[] = new byte[maxBytesInitial];
- int len = abstractFile.read(buffer, 0, maxBytesInitial);
-
- try {
- LinkedHashSet mimeSet = (LinkedHashSet)mimeUtil.getMimeTypesByteArray(buffer);
-
- Iterator it = mimeSet.iterator();
- while (it.hasNext()) {
- MimeType mt = (MimeType)it.next();
- ret.type = mt.getMediaType() + "/" + mt.getSubType();
- break; //just take the first one for now
- }
-
- } catch (MimeException ex) {
- //do nothing
- }
-
- return ret;
-
- } catch (Exception ex) {
- Exceptions.printStackTrace(ex);
- return new FileIdInfo();
- }
- }
-
-}
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java
index f7280fc1e6..b7099c5098 100755
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/AddContentToHashDbAction.java
@@ -82,7 +82,7 @@ final class AddContentToHashDbAction extends AbstractAction implements Presenter
super(SINGLE_SELECTION_NAME);
// Disable the menu if file ingest is in progress.
- if (IngestManager.getDefault().isIngestRunning()) {
+ if (IngestManager.getInstance().isIngestRunning()) {
setEnabled(false);
return;
}
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java
index 7d16a1c006..e623b4f561 100644
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
@@ -42,22 +43,24 @@ import org.sleuthkit.datamodel.TskException;
import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
+import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.datamodel.HashInfo;
public class HashDbIngestModule extends IngestModuleAdapter implements FileIngestModule {
private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
private static final int MAX_COMMENT_SIZE = 500;
- private static int messageId = 0; // RJCTODO: This is not thread safe
- private final IngestServices services = IngestServices.getDefault();
+ private final IngestServices services = IngestServices.getInstance();
private final Hash hasher = new Hash();
private final SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase();
private final HashDbManager hashDbManager = HashDbManager.getInstance();
private final HashLookupModuleSettings settings;
private List knownBadHashSets = new ArrayList<>();
private List knownHashSets = new ArrayList<>();
- private int knownBadCount = 0;
- private long calctime = 0;
- private long lookuptime = 0;
+ private long jobId;
+ private static AtomicLong totalKnownBadCount = new AtomicLong(0);
+ private static AtomicLong totalCalctime = new AtomicLong(0);
+ private static AtomicLong totalLookuptime = new AtomicLong(0);
+ private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
HashDbIngestModule(HashLookupModuleSettings settings) {
this.settings = settings;
@@ -65,30 +68,34 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
@Override
public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException {
+ jobId = context.getJobId();
getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets);
- if (knownBadHashSets.isEmpty()) {
- services.postMessage(IngestMessage.createWarningMessage(++messageId,
- HashLookupModuleFactory.getModuleName(),
- NbBundle.getMessage(this.getClass(),
- "HashDbIngestModule.noKnownBadHashDbSetMsg"),
- NbBundle.getMessage(this.getClass(),
- "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn")));
- }
-
getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets);
- if (knownHashSets.isEmpty()) {
- services.postMessage(IngestMessage.createWarningMessage(++messageId,
- HashLookupModuleFactory.getModuleName(),
- NbBundle.getMessage(this.getClass(),
- "HashDbIngestModule.noKnownHashDbSetMsg"),
- NbBundle.getMessage(this.getClass(),
- "HashDbIngestModule.knownFileSearchWillNotExecuteWarn")));
+
+ if (refCounter.incrementAndGet(jobId) == 1) {
+ // if first module for this job then post error msgs if needed
+
+ if (knownBadHashSets.isEmpty()) {
+ services.postMessage(IngestMessage.createWarningMessage(
+ HashLookupModuleFactory.getModuleName(),
+ NbBundle.getMessage(this.getClass(),
+ "HashDbIngestModule.noKnownBadHashDbSetMsg"),
+ NbBundle.getMessage(this.getClass(),
+ "HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn")));
+ }
+
+ if (knownHashSets.isEmpty()) {
+ services.postMessage(IngestMessage.createWarningMessage(
+ HashLookupModuleFactory.getModuleName(),
+ NbBundle.getMessage(this.getClass(),
+ "HashDbIngestModule.noKnownHashDbSetMsg"),
+ NbBundle.getMessage(this.getClass(),
+ "HashDbIngestModule.knownFileSearchWillNotExecuteWarn")));
+ }
}
}
private void getEnabledHashSets(List hashSets, List enabledHashSets) {
- assert hashSets != null;
- assert enabledHashSets != null;
enabledHashSets.clear();
for (HashDb db : hashSets) {
if (settings.isHashSetEnabled(db.getHashSetName())) {
@@ -123,10 +130,12 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
try {
long calcstart = System.currentTimeMillis();
md5Hash = hasher.calculateMd5(file);
- calctime += (System.currentTimeMillis() - calcstart);
+ long delta = (System.currentTimeMillis() - calcstart);
+ totalCalctime.addAndGet(delta);
+
} catch (IOException ex) {
logger.log(Level.WARNING, "Error calculating hash of file " + name, ex);
- services.postMessage(IngestMessage.createErrorMessage(++messageId,
+ services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.fileReadErrorMsg",
@@ -147,12 +156,13 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
HashInfo hashInfo = db.lookUp(file);
if (null != hashInfo) {
foundBad = true;
- knownBadCount += 1;
+ totalKnownBadCount.incrementAndGet();
+
try {
skCase.setKnown(file, TskData.FileKnown.BAD);
} catch (TskException ex) {
logger.log(Level.WARNING, "Couldn't set known bad state for file " + name + " - see sleuthkit log for details", ex);
- services.postMessage(IngestMessage.createErrorMessage(++messageId,
+ services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg",
@@ -180,10 +190,12 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages());
}
- lookuptime += (System.currentTimeMillis() - lookupstart);
+ long delta = (System.currentTimeMillis() - lookupstart);
+ totalLookuptime.addAndGet(delta);
+
} catch (TskException ex) {
logger.log(Level.WARNING, "Couldn't lookup known bad hash for file " + name + " - see sleuthkit log for details", ex);
- services.postMessage(IngestMessage.createErrorMessage(++messageId,
+ services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg",
@@ -211,10 +223,12 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
ret = ProcessResult.ERROR;
}
}
- lookuptime += (System.currentTimeMillis() - lookupstart);
+ long delta = (System.currentTimeMillis() - lookupstart);
+ totalLookuptime.addAndGet(delta);
+
} catch (TskException ex) {
logger.log(Level.WARNING, "Couldn't lookup known hash for file " + name + " - see sleuthkit log for details", ex);
- services.postMessage(IngestMessage.createErrorMessage(++messageId,
+ services.postMessage(IngestMessage.createErrorMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.hashLookupErrorMsg",
@@ -274,7 +288,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
detailsSb.append("");
- services.postMessage(IngestMessage.createDataMessage(++messageId, HashLookupModuleFactory.getModuleName(),
+ services.postMessage(IngestMessage.createDataMessage( HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.postToBB.knownBadMsg",
abstractFile.getName()),
@@ -288,41 +302,43 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
}
}
-
+
@Override
public void shutDown(boolean ingestJobCancelled) {
- if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
- StringBuilder detailsSb = new StringBuilder();
- //details
- detailsSb.append("");
+ if (refCounter.decrementAndGet(jobId) == 0) {
+ if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
+ StringBuilder detailsSb = new StringBuilder();
+ //details
+ detailsSb.append("");
- detailsSb.append("| ")
- .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.knownBadsFound"))
- .append(" | ");
- detailsSb.append("").append(knownBadCount).append(" | ");
+ detailsSb.append("| ")
+ .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.knownBadsFound"))
+ .append(" | ");
+ detailsSb.append("").append(totalKnownBadCount.get()).append(" | ");
- detailsSb.append("| ")
- .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalCalcTime"))
- .append(" | ").append(calctime).append(" | \n");
- detailsSb.append("| ")
- .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalLookupTime"))
- .append(" | ").append(lookuptime).append(" | \n");
- detailsSb.append(" ");
+ detailsSb.append("| ")
+ .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalCalcTime"))
+ .append(" | ").append(totalCalctime.get()).append(" | \n");
+ detailsSb.append("| ")
+ .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.totalLookupTime"))
+ .append(" | ").append(totalLookuptime.get()).append(" | \n");
+ detailsSb.append(" ");
- detailsSb.append("")
- .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.databasesUsed"))
- .append(" \n");
- for (HashDb db : knownBadHashSets) {
- detailsSb.append("- ").append(db.getHashSetName()).append("
\n");
+ detailsSb.append("")
+ .append(NbBundle.getMessage(this.getClass(), "HashDbIngestModule.complete.databasesUsed"))
+ .append(" \n");
+ for (HashDb db : knownBadHashSets) {
+ detailsSb.append("- ").append(db.getHashSetName()).append("
\n");
+ }
+
+ detailsSb.append(" ");
+ services.postMessage(IngestMessage.createMessage(
+ IngestMessage.MessageType.INFO,
+ HashLookupModuleFactory.getModuleName(),
+ NbBundle.getMessage(this.getClass(),
+ "HashDbIngestModule.complete.hashLookupResults"),
+ detailsSb.toString()));
}
-
- detailsSb.append(" ");
- services.postMessage(IngestMessage.createMessage(++messageId,
- IngestMessage.MessageType.INFO,
- HashLookupModuleFactory.getModuleName(),
- NbBundle.getMessage(this.getClass(),
- "HashDbIngestModule.complete.hashLookupResults"),
- detailsSb.toString()));
}
}
}
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java
index e97dfbe4bb..514b99b0ca 100755
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbManager.java
@@ -94,7 +94,7 @@ public class HashDbManager implements PropertyChangeListener {
*/
public enum SetEvt {
- DB_ADDED, DB_DELETED
+ DB_ADDED, DB_DELETED, DB_INDEXED
};
/**
@@ -345,7 +345,7 @@ public class HashDbManager implements PropertyChangeListener {
*/
public synchronized void removeHashDatabase(HashDb hashDb) throws HashDbManagerException {
// Don't remove a database if ingest is running
- boolean ingestIsRunning = IngestManager.getDefault().isIngestRunning();
+ boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning();
if (ingestIsRunning) {
throw new HashDbManagerException(NbBundle.getMessage(this.getClass(), "HashDbManager.ingestRunningExceptionMsg"));
}
@@ -1019,6 +1019,7 @@ public class HashDbManager implements PropertyChangeListener {
try {
hashDb.propertyChangeSupport.firePropertyChange(HashDb.Event.INDEXING_DONE.toString(), null, hashDb);
+ hashDb.propertyChangeSupport.firePropertyChange(HashDbManager.SetEvt.DB_INDEXED.toString(), null, hashDb.getHashSetName());
} catch (Exception e) {
logger.log(Level.SEVERE, "HashDbManager listener threw exception", e);
MessageNotifyUtil.Notify.show(
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSearchPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSearchPanel.java
index 9df96dd09e..dd23ce2c78 100644
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSearchPanel.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSearchPanel.java
@@ -55,7 +55,7 @@ import org.sleuthkit.autopsy.ingest.IngestManager;
* Check if ingest is currently running and refresh the panel accordingly.
*/
public void refresh() {
- boolean running = IngestManager.getDefault().isIngestRunning();
+ boolean running = IngestManager.getInstance().isIngestRunning();
if(running) {
titleLabel.setForeground(Color.red);
titleLabel.setText(NbBundle.getMessage(this.getClass(), "HashDbSearchPanel.titleText.ingestOngoing"));
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleFactory.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleFactory.java
index a57ac7677c..3e2934f323 100755
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleFactory.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleFactory.java
@@ -58,35 +58,37 @@ public class HashLookupModuleFactory extends IngestModuleFactoryAdapter {
}
@Override
- public IngestModuleIngestJobSettings getDefaultModuleSettings() {
+ public IngestModuleIngestJobSettings getDefaultIngestJobSettings() {
+ // All available hash sets are enabled by default.
HashDbManager hashDbManager = HashDbManager.getInstance();
- List enabledHashSets = new ArrayList<>();
- List knownFileHashSets = hashDbManager.getKnownFileHashSets();
- for (HashDbManager.HashDb db : knownFileHashSets) {
- if (db.getSearchDuringIngest()) {
- enabledHashSets.add(db.getHashSetName());
- }
+ List knownHashSetNames = getHashSetNames(hashDbManager.getKnownFileHashSets());
+ List knownBadHashSetNames = getHashSetNames(hashDbManager.getKnownBadFileHashSets());
+ return new HashLookupModuleSettings(hashDbManager.getAlwaysCalculateHashes(), knownHashSetNames, knownBadHashSetNames);
+ }
+
+ private List getHashSetNames(List hashDbs) {
+ List hashSetNames = new ArrayList<>();
+ for (HashDbManager.HashDb db : hashDbs) {
+ hashSetNames.add(db.getHashSetName());
}
- List knownBadFileHashSets = hashDbManager.getKnownBadFileHashSets();
- for (HashDbManager.HashDb db : knownBadFileHashSets) {
- if (db.getSearchDuringIngest()) {
- enabledHashSets.add(db.getHashSetName());
- }
- }
- return new HashLookupModuleSettings(hashDbManager.getAlwaysCalculateHashes(), enabledHashSets);
+ return hashSetNames;
}
@Override
- public boolean hasModuleSettingsPanel() {
+ public boolean hasIngestJobSettingsPanel() {
return true;
}
@Override
- public IngestModuleIngestJobSettingsPanel getModuleSettingsPanel(IngestModuleIngestJobSettings settings) {
- if (moduleSettingsPanel == null) {
- moduleSettingsPanel = new HashLookupModuleSettingsPanel();
+ public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) {
+ if (!(settings instanceof HashLookupModuleSettings)) {
+ throw new IllegalArgumentException("Expected settings argument to be instanceof HashLookupModuleSettings");
+ }
+ if (moduleSettingsPanel == null) {
+ moduleSettingsPanel = new HashLookupModuleSettingsPanel((HashLookupModuleSettings) settings);
+ } else {
+ moduleSettingsPanel.reset((HashLookupModuleSettings) settings);
}
- moduleSettingsPanel.load(); // RJCTODO: Fix this, use passed in settings
return moduleSettingsPanel;
}
@@ -109,7 +111,6 @@ public class HashLookupModuleFactory extends IngestModuleFactoryAdapter {
@Override
public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) {
- assert settings instanceof HashLookupModuleSettings;
if (!(settings instanceof HashLookupModuleSettings)) {
throw new IllegalArgumentException("Expected settings argument to be instanceof HashLookupModuleSettings");
}
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettings.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettings.java
index 781d40fee7..38ceaa846b 100755
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettings.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettings.java
@@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.hashdatabase;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
@@ -27,21 +28,29 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
*/
final class HashLookupModuleSettings implements IngestModuleIngestJobSettings {
- private final HashSet enabledHashSets = new HashSet<>();
+ private final HashSet namesOfEnabledKnownHashSets = new HashSet<>();
+ private final HashSet namesOfEnabledKnownBadHashSets = new HashSet<>();
private boolean shouldCalculateHashes = true;
- HashLookupModuleSettings(boolean shouldCalculateHashes, List enabledHashSetNames) {
+ HashLookupModuleSettings(boolean shouldCalculateHashes, List namesOfEnabledKnownHashSets, List namesOfEnabledKnownBadHashSets) {
this.shouldCalculateHashes = shouldCalculateHashes;
- for (String hashSet : enabledHashSetNames) {
- enabledHashSets.add(hashSet);
- }
+ this.namesOfEnabledKnownHashSets.addAll(namesOfEnabledKnownHashSets);
+ this.namesOfEnabledKnownBadHashSets.addAll(namesOfEnabledKnownBadHashSets);
}
boolean shouldCalculateHashes() {
return shouldCalculateHashes;
}
-
+
boolean isHashSetEnabled(String hashSetName) {
- return enabledHashSets.contains(hashSetName);
+ return (namesOfEnabledKnownHashSets.contains(hashSetName) || namesOfEnabledKnownBadHashSets.contains(hashSetName));
+ }
+
+ List getNamesOfEnabledKnownHashSets() {
+ return new ArrayList<>(namesOfEnabledKnownHashSets);
+ }
+
+ List getNamesOfEnabledKnownBadHashSets() {
+ return new ArrayList<>(namesOfEnabledKnownBadHashSets);
}
}
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.java
index 35cfd666ae..6d83818961 100644
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupModuleSettingsPanel.java
@@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
- * Copyright 2011 - 2014 Basis Technology Corp.
+ * Copyright 2011-2014 Basis Technology Corp.
* Contact: carrier sleuthkit org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,51 +21,61 @@ package org.sleuthkit.autopsy.hashdatabase;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.logging.Level;
-import javax.swing.JOptionPane;
import javax.swing.JScrollPane;
import javax.swing.JTable;
import javax.swing.table.AbstractTableModel;
import javax.swing.table.TableColumn;
-import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.autopsy.hashdatabase.HashDbManager.HashDb;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
/**
- * Instances of this class provide a simplified UI for managing the hash sets
- * configuration.
+ * Ingest job settings panel for hash lookup file ingest modules.
*/
-public class HashLookupModuleSettingsPanel extends IngestModuleIngestJobSettingsPanel implements PropertyChangeListener {
+public final class HashLookupModuleSettingsPanel extends IngestModuleIngestJobSettingsPanel implements PropertyChangeListener {
private final HashDbManager hashDbManager = HashDbManager.getInstance();
- private HashDatabasesTableModel knownTableModel;
- private HashDatabasesTableModel knownBadTableModel;
+ private final List knownHashSetModels = new ArrayList<>();
+ private final HashSetsTableModel knownHashSetsTableModel = new HashSetsTableModel(knownHashSetModels);
+ private final List knownBadHashSetModels = new ArrayList<>();
+ private final HashSetsTableModel knownBadHashSetsTableModel = new HashSetsTableModel(knownBadHashSetModels);
- HashLookupModuleSettingsPanel() {
- knownTableModel = new HashDatabasesTableModel(HashDbManager.HashDb.KnownFilesType.KNOWN);
- knownBadTableModel = new HashDatabasesTableModel(HashDbManager.HashDb.KnownFilesType.KNOWN_BAD);
+ HashLookupModuleSettingsPanel(HashLookupModuleSettings settings) {
+ initializeHashSetModels(settings);
initComponents();
customizeComponents();
}
+ private void initializeHashSetModels(HashLookupModuleSettings settings) {
+ initializeHashSetModels(settings, hashDbManager.getKnownFileHashSets(), knownHashSetModels);
+ initializeHashSetModels(settings, hashDbManager.getKnownBadFileHashSets(), knownBadHashSetModels);
+ }
+
+ private void initializeHashSetModels(HashLookupModuleSettings settings, List hashDbs, List hashSetModels) {
+ hashSetModels.clear();
+ for (HashDb db : hashDbs) {
+ String name = db.getHashSetName();
+ hashSetModels.add(new HashSetModel(name, settings.isHashSetEnabled(name), isHashDbIndexed(db)));
+ }
+ }
+
private void customizeComponents() {
- customizeHashDbsTable(jScrollPane1, knownHashTable, knownTableModel);
- customizeHashDbsTable(jScrollPane2, knownBadHashTable, knownBadTableModel);
+ customizeHashSetsTable(jScrollPane1, knownHashTable, knownHashSetsTableModel);
+ customizeHashSetsTable(jScrollPane2, knownBadHashTable, knownBadHashSetsTableModel);
alwaysCalcHashesCheckbox.setSelected(hashDbManager.getAlwaysCalculateHashes());
- load();
hashDbManager.addPropertyChangeListener(this);
}
- private void customizeHashDbsTable(JScrollPane scrollPane, JTable table, HashDatabasesTableModel tableModel) {
+ private void customizeHashSetsTable(JScrollPane scrollPane, JTable table, HashSetsTableModel tableModel) {
table.setModel(tableModel);
table.setTableHeader(null);
table.setRowSelectionAllowed(false);
-
final int width1 = scrollPane.getPreferredSize().width;
knownHashTable.setAutoResizeMode(JTable.AUTO_RESIZE_NEXT_COLUMN);
TableColumn column;
@@ -81,65 +91,132 @@ public class HashLookupModuleSettingsPanel extends IngestModuleIngestJobSettings
@Override
public void propertyChange(PropertyChangeEvent event) {
- if (event.getPropertyName().equals(HashDbManager.SetEvt.DB_ADDED.name()) ||
- event.getPropertyName().equals(HashDbManager.SetEvt.DB_DELETED.name())) {
- load();
+ if (event.getPropertyName().equals(HashDbManager.SetEvt.DB_ADDED.name())
+ || event.getPropertyName().equals(HashDbManager.SetEvt.DB_DELETED.name())
+ || event.getPropertyName().equals(HashDbManager.SetEvt.DB_INDEXED.name())) {
+ update();
}
}
-
+
@Override
public IngestModuleIngestJobSettings getSettings() {
- List enabledHashSets = new ArrayList<>();
- List knownFileHashSets = hashDbManager.getKnownFileHashSets();
- for (HashDb db : knownFileHashSets) {
- if (db.getSearchDuringIngest()) {
- enabledHashSets.add(db.getHashSetName());
+ return new HashLookupModuleSettings(alwaysCalcHashesCheckbox.isSelected(),
+ getNamesOfEnabledHashSets(knownHashSetModels),
+ getNamesOfEnabledHashSets(knownBadHashSetModels));
+ }
+
+ private List getNamesOfEnabledHashSets(List hashSetModels) {
+ List namesOfEnabledHashSets = new ArrayList<>();
+ for (HashSetModel model : hashSetModels) {
+ if (model.isEnabled() && model.isIndexed()) {
+ namesOfEnabledHashSets.add(model.getName());
}
}
- List knownBadFileHashSets = hashDbManager.getKnownBadFileHashSets();
- for (HashDb db : knownBadFileHashSets) {
- if (db.getSearchDuringIngest()) {
- enabledHashSets.add(db.getHashSetName());
- }
- }
- return new HashLookupModuleSettings(alwaysCalcHashesCheckbox.isSelected(), enabledHashSets);
+ return namesOfEnabledHashSets;
}
- void load() {
- knownTableModel.load();
- knownBadTableModel.load();
+ void update() {
+ updateHashSetModels();
+ knownHashSetsTableModel.fireTableDataChanged();
+ knownBadHashSetsTableModel.fireTableDataChanged();
}
- void store() {
- hashDbManager.save();
+ private void updateHashSetModels() {
+ updateHashSetModels(hashDbManager.getKnownFileHashSets(), knownHashSetModels);
+ updateHashSetModels(hashDbManager.getKnownBadFileHashSets(), knownBadHashSetModels);
}
- private class HashDatabasesTableModel extends AbstractTableModel {
-
- private final HashDbManager.HashDb.KnownFilesType hashDatabasesType;
- private List hashDatabases;
-
- HashDatabasesTableModel(HashDbManager.HashDb.KnownFilesType hashDatabasesType) {
- this.hashDatabasesType = hashDatabasesType;
- getHashDatabases();
+ void updateHashSetModels(List hashDbs, List hashSetModels) {
+ Map hashSetDbs = new HashMap<>();
+ for (HashDb db : hashDbs) {
+ hashSetDbs.put(db.getHashSetName(), db);
}
- private void getHashDatabases() {
- if (HashDbManager.HashDb.KnownFilesType.KNOWN == hashDatabasesType) {
- hashDatabases = hashDbManager.getKnownFileHashSets();
+ // Update the hash sets and detect deletions.
+ List deletedHashSetModels = new ArrayList<>();
+ for (HashSetModel model : hashSetModels) {
+ String hashSetName = model.getName();
+ if (hashSetDbs.containsKey(hashSetName)) {
+ HashDb db = hashSetDbs.get(hashSetName);
+ model.setIndexed(isHashDbIndexed(db));
+ hashSetDbs.remove(hashSetName);
} else {
- hashDatabases = hashDbManager.getKnownBadFileHashSets();
+ deletedHashSetModels.add(model);
}
}
- private void load() {
- getHashDatabases();
- fireTableDataChanged();
+ // Remove the deleted hash sets.
+ for (HashSetModel model : deletedHashSetModels) {
+ hashSetModels.remove(model);
+ }
+
+ // Add any new hash sets. All new sets are enabled by default.
+ for (HashDb db : hashSetDbs.values()) {
+ String name = db.getHashSetName();
+ hashSetModels.add(new HashSetModel(name, true, isHashDbIndexed(db)));
+ }
+ }
+
+ void reset(HashLookupModuleSettings newSettings) {
+ initializeHashSetModels(newSettings);
+ knownHashSetsTableModel.fireTableDataChanged();
+ knownBadHashSetsTableModel.fireTableDataChanged();
+ }
+
+ private boolean isHashDbIndexed(HashDb hashDb) {
+ boolean indexed = false;
+ try {
+ indexed = hashDb.hasIndex();
+ } catch (TskCoreException ex) {
+ Logger.getLogger(HashLookupModuleSettingsPanel.class.getName()).log(Level.SEVERE, "Error getting indexed status info for hash set (name = " + hashDb.getHashSetName() + ")", ex);
+ }
+ return indexed;
+ }
+
+ private static final class HashSetModel {
+
+ private final String name;
+ private boolean indexed;
+ private boolean enabled;
+
+ HashSetModel(String name, boolean enabled, boolean indexed) {
+ this.name = name;
+ this.enabled = enabled;
+ this.indexed = indexed;
+ }
+
+ String getName() {
+ return name;
+ }
+
+ void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ boolean isEnabled() {
+ return enabled;
+ }
+
+ void setIndexed(boolean indexed) {
+ this.indexed = indexed;
+ }
+
+ boolean isIndexed() {
+ return indexed;
+ }
+ }
+
+ private static final class HashSetsTableModel extends AbstractTableModel {
+
+ private final List hashSets;
+
+ HashSetsTableModel(List hashSets) {
+ this.hashSets = hashSets;
}
@Override
public int getRowCount() {
- return hashDatabases.size();
+ return hashSets.size();
}
@Override
@@ -149,36 +226,22 @@ public class HashLookupModuleSettingsPanel extends IngestModuleIngestJobSettings
@Override
public Object getValueAt(int rowIndex, int columnIndex) {
- HashDb db = hashDatabases.get(rowIndex);
if (columnIndex == 0) {
- return db.getSearchDuringIngest();
+ return hashSets.get(rowIndex).isEnabled();
} else {
- return db.getHashSetName();
+ return hashSets.get(rowIndex).getName();
}
}
@Override
public boolean isCellEditable(int rowIndex, int columnIndex) {
- return !IngestManager.getDefault().isIngestRunning() && columnIndex == 0;
+ return (columnIndex == 0 && hashSets.get(rowIndex).isIndexed());
}
@Override
public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
if (columnIndex == 0) {
- HashDb db = hashDatabases.get(rowIndex);
- boolean dbHasIndex = false;
- try {
- dbHasIndex = db.hasIndex();
- } catch (TskCoreException ex) {
- Logger.getLogger(HashLookupModuleSettingsPanel.class.getName()).log(Level.SEVERE, "Error getting info for " + db.getHashSetName() + " hash database", ex);
- }
- if (((Boolean) getValueAt(rowIndex, columnIndex)) || dbHasIndex) {
- db.setSearchDuringIngest((Boolean) aValue);
- } else {
- JOptionPane.showMessageDialog(HashLookupModuleSettingsPanel.this,
- NbBundle.getMessage(this.getClass(),
- "HashDbSimpleConfigPanel.dlgMsg.mustIndexDbBeforeUse"));
- }
+ hashSets.get(rowIndex).setEnabled((Boolean) aValue);
}
}
diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java
index c84dd442b1..4e1e9ceb70 100644
--- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java
+++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java
@@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.hashdatabase;
import java.awt.Color;
import java.awt.Component;
+import java.awt.EventQueue;
import java.awt.Frame;
import java.awt.event.KeyEvent;
import java.beans.PropertyChangeEvent;
@@ -71,8 +72,13 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
- if (isFileIngestStatusChangeEvent(evt)) {
- updateComponents();
+ if (isIngestJobEvent(evt)) {
+ EventQueue.invokeLater(new Runnable() {
+ @Override
+ public void run() {
+ updateComponents();
+ }
+ });
}
}
});
@@ -105,7 +111,7 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
}
private void updateComponentsForNoSelection() {
- boolean ingestIsRunning = IngestManager.getDefault().isIngestRunning();
+ boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning();
// Update descriptive labels.
hashDbNameLabel.setText(NO_SELECTION_TEXT);
@@ -135,7 +141,7 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
}
private void updateComponentsForSelection(HashDb db) {
- boolean ingestIsRunning = IngestManager.getDefault().isIngestRunning();
+ boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning();
// Update descriptive labels.
hashDbNameLabel.setText(db.getHashSetName());
@@ -224,8 +230,10 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa
return shortenedPath;
}
- private boolean isFileIngestStatusChangeEvent(PropertyChangeEvent evt) {
- return evt.getPropertyName().equals(IngestManager.IngestEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.COMPLETED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.STOPPED.toString());
+ private boolean isIngestJobEvent(PropertyChangeEvent evt) {
+ return evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_STARTED.toString())
+ || evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_COMPLETED.toString())
+ || evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_CANCELLED.toString());
}
@Override
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java
index e013ec2fac..7af85f7e3f 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileHtmlExtract.java
@@ -177,10 +177,6 @@ import org.sleuthkit.datamodel.ReadContentInputStream;
+ sourceFile.getName() + "' (id: " + sourceFile.getId() + ").", ingEx);
throw ingEx; //need to rethrow/return to signal error and move on
}
-
- //check if need invoke commit/search between chunks
- //not to delay commit if timer has gone off
- module.checkRunCommitSearch();
}
} catch (IOException ex) {
logger.log(Level.WARNING, "Unable to read content stream from " + sourceFile.getId() + ": " + sourceFile.getName(), ex);
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java
index 4d82827c6a..6f96ff2495 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileStringExtract.java
@@ -147,10 +147,6 @@ class AbstractFileStringExtract implements AbstractFileExtract {
throw ingEx; //need to rethrow/return to signal error and move on
}
- //check if need invoke commit/search between chunks
- //not to delay commit if timer has gone off
- module.checkRunCommitSearch();
-
//debug.close();
}
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java
index 4f52805753..e19be18a5c 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileTikaTextExtract.java
@@ -223,10 +223,6 @@ class AbstractFileTikaTextExtract implements AbstractFileExtract {
+ sourceFile.getName() + "' (id: " + sourceFile.getId() + ").", ingEx);
throw ingEx; //need to rethrow/return to signal error and move on
}
-
- //check if need invoke commit/search between chunks
- //not to delay commit if timer has gone off
- module.checkRunCommitSearch();
}
} catch (IOException ex) {
final String msg = "Exception: Unable to read Tika content stream from " + sourceFile.getId() + ": " + sourceFile.getName();
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java
index 54779a4dab..b69eef4fe6 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractKeywordSearchPerformer.java
@@ -86,7 +86,7 @@ abstract class AbstractKeywordSearchPerformer extends javax.swing.JPanel impleme
@Override
public void search() {
- boolean isIngestRunning = IngestManager.getDefault().isIngestRunning();
+ boolean isIngestRunning = IngestManager.getInstance().isIngestRunning();
if (filesIndexed == 0) {
if (isIngestRunning) {
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
index 040e9e0c03..a608ce590f 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
@@ -263,3 +263,10 @@ KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster
KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index:
KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer)
KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default)
+KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit:
+KeywordSearchIngestModule.kwHitLbl=Keyword hit:
+KeywordSearchIngestModule.kwHitThLbl=Keyword
+KeywordSearchIngestModule.previewThLbl=Preview
+KeywordSearchIngestModule.fileThLbl=File
+KeywordSearchIngestModule.listThLbl=List
+KeywordSearchIngestModule.regExThLbl=Reg Ex
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java
index 12d9999b01..bdb304dabf 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java
@@ -53,7 +53,6 @@ import org.sleuthkit.datamodel.ContentVisitor;
import org.sleuthkit.datamodel.DerivedFile;
import org.sleuthkit.datamodel.Directory;
import org.sleuthkit.datamodel.File;
-import org.sleuthkit.datamodel.FsContent;
import org.sleuthkit.datamodel.LayoutFile;
import org.sleuthkit.datamodel.LocalFile;
import org.sleuthkit.datamodel.ReadContentInputStream;
@@ -66,7 +65,7 @@ import org.sleuthkit.datamodel.TskCoreException;
class Ingester {
private static final Logger logger = Logger.getLogger(Ingester.class.getName());
- private boolean uncommitedIngests = false;
+ private volatile boolean uncommitedIngests = false;
private final ExecutorService upRequestExecutor = Executors.newSingleThreadExecutor();
private final Server solrServer = KeywordSearch.getServer();
private final GetContentFieldsV getContentFieldsV = new GetContentFieldsV();
@@ -74,8 +73,7 @@ class Ingester {
//for ingesting chunk as SolrInputDocument (non-content-streaming, by-pass tika)
//TODO use a streaming way to add content to /update handler
- private final static int MAX_DOC_CHUNK_SIZE = 1024*1024;
- private final byte[] docChunkContentBuf = new byte[MAX_DOC_CHUNK_SIZE];
+ private static final int MAX_DOC_CHUNK_SIZE = 1024*1024;
private static final String docContentEncoding = "UTF-8";
@@ -286,6 +284,7 @@ class Ingester {
throw new IngesterException(msg);
}
+ final byte[] docChunkContentBuf = new byte[MAX_DOC_CHUNK_SIZE];
SolrInputDocument updateDoc = new SolrInputDocument();
for (String key : fields.keySet()) {
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form
index 1f31418be7..83675f4797 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.form
@@ -39,14 +39,6 @@
-
-
-
-
-
-
-
-
@@ -67,7 +59,10 @@
-
+
+
+
+
@@ -114,7 +109,7 @@
-
+
@@ -128,8 +123,9 @@
-
-
+
+
+
@@ -146,15 +142,15 @@
-
+
-
+
-
+
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java
index 4d0d3f75c0..27ba1933e5 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java
@@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
+import java.awt.EventQueue;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.beans.PropertyChangeEvent;
@@ -52,7 +53,6 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
private static Logger logger = Logger.getLogger(KeywordSearchEditListPanel.class.getName());
private KeywordTableModel tableModel;
private KeywordList currentKeywordList;
- private boolean ingestRunning;
/**
* Creates new form KeywordSearchEditListPanel
@@ -101,7 +101,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
}
});
- initButtons();
+ setButtonStates();
addWordField.setComponentPopupMenu(rightClickMenu);
ActionListener actList = new ActionListener() {
@@ -124,49 +124,28 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
pasteMenuItem.addActionListener(actList);
selectAllMenuItem.addActionListener(actList);
- if (IngestManager.getDefault().isIngestRunning()) {
- initIngest(0);
- } else {
- initIngest(1);
- }
+ setButtonStates();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName();
- Object oldValue = evt.getOldValue();
- if (changed.equals(IngestEvent.COMPLETED.toString())
- && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
- initIngest(1);
- } else if (changed.equals(IngestEvent.STARTED.toString())
- && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
- initIngest(0);
- } else if (changed.equals(IngestEvent.STOPPED.toString())
- && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) {
- initIngest(1);
+ if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString())
+ || changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString())
+ || changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) {
+ EventQueue.invokeLater(new Runnable() {
+ @Override
+ public void run() {
+ setButtonStates();
+ }
+ });
}
}
});
}
- /**
- * Initialize this panel depending on whether ingest is running
- *
- * @param running case 0: ingest running case 1: ingest not running
- */
- private void initIngest(int running) {
- switch (running) {
- case 0:
- ingestRunning = true;
- break;
- case 1:
- ingestRunning = false;
- break;
- }
- initButtons();
- }
-
- void initButtons() {
+ void setButtonStates() {
+ boolean ingestRunning = IngestManager.getInstance().isIngestRunning();
boolean listSet = currentKeywordList != null;
boolean isLocked = !listSet ? true : currentKeywordList.isLocked();
boolean noKeywords = !listSet ? true : currentKeywordList.getKeywords().isEmpty();
@@ -237,9 +216,6 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
selectAllMenuItem.setText(org.openide.util.NbBundle.getMessage(KeywordSearchEditListPanel.class, "KeywordSearchEditListPanel.selectAllMenuItem.text")); // NOI18N
rightClickMenu.add(selectAllMenuItem);
- setMinimumSize(new java.awt.Dimension(340, 300));
- setPreferredSize(new java.awt.Dimension(340, 420));
-
jScrollPane1.setPreferredSize(new java.awt.Dimension(340, 300));
keywordTable.setModel(tableModel);
@@ -363,7 +339,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
.addComponent(saveListButton)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(deleteListButton)))))
- .addGap(0, 44, Short.MAX_VALUE)))
+ .addGap(0, 0, Short.MAX_VALUE)))
.addContainerGap())))
);
listEditorPanelLayout.setVerticalGroup(
@@ -372,7 +348,8 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
.addContainerGap()
.addComponent(keywordsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
- .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 188, javax.swing.GroupLayout.PREFERRED_SIZE)
+ .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 178, javax.swing.GroupLayout.PREFERRED_SIZE)
+ .addGap(10, 10, 10)
.addGroup(listEditorPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(listEditorPanelLayout.createSequentialGroup()
.addGroup(listEditorPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
@@ -385,14 +362,14 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
.addGroup(listEditorPanelLayout.createSequentialGroup()
.addGap(123, 123, 123)
.addComponent(listOptionsSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 6, javax.swing.GroupLayout.PREFERRED_SIZE)))
- .addGap(18, 18, 18)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(ingestMessagesCheckbox)
- .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 13, Short.MAX_VALUE)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(listEditorPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(exportButton)
.addComponent(saveListButton)
.addComponent(deleteListButton))
- .addGap(42, 42, 42))
+ .addContainerGap())
);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
@@ -403,7 +380,9 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addComponent(listEditorPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
+ .addGroup(layout.createSequentialGroup()
+ .addComponent(listEditorPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
+ .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
);
}// //GEN-END:initComponents
@@ -441,7 +420,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
chRegex.setSelected(false);
addWordField.setText("");
- initButtons();
+ setButtonStates();
}//GEN-LAST:event_addWordButtonActionPerformed
private void deleteWordButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteWordButtonActionPerformed
@@ -449,7 +428,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
tableModel.deleteSelected(keywordTable.getSelectedRows());
KeywordSearchListsXML.getCurrent().addList(currentKeywordList);
- initButtons();
+ setButtonStates();
}
}//GEN-LAST:event_deleteWordButtonActionPerformed
@@ -549,11 +528,11 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
currentKeywordList = loader.getListsL(false).get(index);
tableModel.resync();
- initButtons();
+ setButtonStates();
} else {
currentKeywordList = null;
tableModel.resync();
- initButtons();
+ setButtonStates();
}
}
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalLanguageSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalLanguageSettingsPanel.java
index 5fc9649427..b325d566a0 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalLanguageSettingsPanel.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalLanguageSettingsPanel.java
@@ -130,7 +130,7 @@ class KeywordSearchGlobalLanguageSettingsPanel extends javax.swing.JPanel implem
enableUTF8Checkbox.setSelected(utf8);
final boolean extractEnabled = utf16 || utf8;
- boolean ingestNotRunning = !IngestManager.getDefault().isIngestRunning() && !IngestManager.getDefault().isIngestRunning();
+ boolean ingestNotRunning = !IngestManager.getInstance().isIngestRunning() && !IngestManager.getInstance().isIngestRunning();
//enable / disable checboxes
activateScriptsCheckboxes(extractEnabled && ingestNotRunning);
enableUTF16Checkbox.setEnabled(ingestNotRunning);
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form
index bad2ca29c9..e0660ba6b1 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.form
@@ -16,7 +16,7 @@
-
+
@@ -57,6 +57,11 @@
+
+
+
+
+
@@ -66,7 +71,7 @@
-
+
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java
index 36b22ef853..ac8e74212d 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java
@@ -44,7 +44,7 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl
if (KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.title"), NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.body"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) {
String toDelete = editListPanel.getCurrentKeywordList().getName();
editListPanel.setCurrentKeywordList(null);
- editListPanel.initButtons();
+ editListPanel.setButtonStates();
// RJCTODO: Move this into a deleteList method in the manager
KeywordSearchListsXML deleter = KeywordSearchListsXML.getCurrent();
deleter.deleteList(toDelete);
@@ -152,11 +152,13 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl
mainSplitPane.setLeftComponent(leftPanel);
+ rightPanel.setPreferredSize(new java.awt.Dimension(360, 327));
+
javax.swing.GroupLayout rightPanelLayout = new javax.swing.GroupLayout(rightPanel);
rightPanel.setLayout(rightPanelLayout);
rightPanelLayout.setHorizontalGroup(
rightPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addGap(0, 318, Short.MAX_VALUE)
+ .addGap(0, 385, Short.MAX_VALUE)
);
rightPanelLayout.setVerticalGroup(
rightPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@@ -169,7 +171,7 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addComponent(mainSplitPane)
+ .addComponent(mainSplitPane, javax.swing.GroupLayout.DEFAULT_SIZE, 665, Short.MAX_VALUE)
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java
index b61601aa93..1bc9cccc73 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java
@@ -44,7 +44,7 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
private void activateWidgets() {
skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown());
showSnippetsCB.setSelected(KeywordSearchSettings.getShowSnippets());
- boolean enable = !IngestManager.getDefault().isIngestRunning();
+ boolean enable = !IngestManager.getInstance().isIngestRunning();
skipNSRLCheckBox.setEnabled(enable);
setTimeSettingEnabled(enable);
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form
index 6a22cf2579..a660de43de 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.form
@@ -16,12 +16,12 @@
-
+
-
+
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java
index 44e1dc22a1..d400abe80c 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSettingsPanel.java
@@ -63,11 +63,11 @@ final class KeywordSearchGlobalSettingsPanel extends IngestModuleGlobalSetttings
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 400, Short.MAX_VALUE)
+ .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 670, Short.MAX_VALUE)
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 300, Short.MAX_VALUE)
+ .addComponent(tabbedPane, javax.swing.GroupLayout.DEFAULT_SIZE, 310, Short.MAX_VALUE)
);
}// //GEN-END:initComponents
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
old mode 100755
new mode 100644
index bba7b7fbe6..dbb9c198e5
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
@@ -18,36 +18,19 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
-import java.awt.event.ActionEvent;
-import java.awt.event.ActionListener;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
-import javax.swing.SwingUtilities;
-import javax.swing.SwingWorker;
-import javax.swing.Timer;
import org.apache.tika.Tika;
-import org.netbeans.api.progress.aggregate.AggregateProgressFactory;
-import org.netbeans.api.progress.aggregate.AggregateProgressHandle;
-import org.netbeans.api.progress.aggregate.ProgressContributor;
-import org.openide.util.Cancellable;
import org.sleuthkit.autopsy.casemodule.Case;
-import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
-import org.sleuthkit.autopsy.coreutils.StopWatch;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestServices;
@@ -55,11 +38,8 @@ import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
-import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
+import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
-import org.sleuthkit.datamodel.BlackboardArtifact;
-import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
-import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.SleuthkitCase;
@@ -95,31 +75,25 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
}
};
private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName());
- private IngestServices services = IngestServices.getDefault();
+ private IngestServices services = IngestServices.getInstance();
private Ingester ingester = null;
- private volatile boolean commitIndex = false; //whether to commit index next time
- private volatile boolean runSearcher = false; //whether to run searcher next time
- private Timer commitTimer;
- private Timer searchTimer;
private Indexer indexer;
- private Searcher currentSearcher;
- private Searcher finalSearcher;
- private volatile boolean searcherDone = true; //mark as done, until it's inited
- private Map> currentResults;
//only search images from current ingest, not images previously ingested/indexed
//accessed read-only by searcher thread
- private Set curDataSourceIds;
- private static final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy
- private static final Lock searcherLock = rwLock.writeLock();
- private volatile int messageID = 0; // RJCTODO: Despite volatile, this is not thread safe, uses increment (not atomic)
- private boolean processedFiles;
+
+ private boolean startedSearching = false;
private SleuthkitCase caseHandle = null;
- private static List textExtractors;
- private static AbstractFileStringExtract stringExtractor;
+ private List textExtractors;
+ private AbstractFileStringExtract stringExtractor;
private final KeywordSearchJobSettings settings;
private boolean initialized = false;
private Tika tikaFormatDetector;
-
+ private long jobId;
+ private long dataSourceId;
+ private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging
+ private int instanceNum = 0;
+ private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
+
private enum IngestStatus {
TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested
@@ -129,10 +103,17 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
SKIPPED_ERROR_TEXTEXTRACT, ///< File was skipped because of text extraction issues
SKIPPED_ERROR_IO ///< File was skipped because of IO issues reading it
};
- private Map ingestStatus;
+ private static final Map ingestStatus = new HashMap<>(); //guarded by itself
+ static void putIngestStatus(long id, IngestStatus status) {
+ synchronized(ingestStatus) {
+ ingestStatus.put(id, status);
+ }
+ }
+
KeywordSearchIngestModule(KeywordSearchJobSettings settings) {
this.settings = settings;
+ instanceNum = instanceCount.getAndIncrement();
}
/**
@@ -142,42 +123,58 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
*/
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
- logger.log(Level.INFO, "init()");
- initialized = false;
-
+ logger.log(Level.INFO, "Initializing instance {0}", instanceNum);
+ initialized = false;
+ jobId = context.getJobId();
caseHandle = Case.getCurrentCase().getSleuthkitCase();
-
tikaFormatDetector = new Tika();
-
ingester = Server.getIngester();
- final Server server = KeywordSearch.getServer();
- try {
- if (!server.isRunning()) {
+ // increment the module reference count
+ // if first instance of this module for this job then check the server and existence of keywords
+ if (refCounter.incrementAndGet(jobId) == 1) {
+ final Server server = KeywordSearch.getServer();
+ try {
+ if (!server.isRunning()) {
+ String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg");
+ logger.log(Level.SEVERE, msg);
+ String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg);
+ services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details));
+ throw new IngestModuleException(msg);
+ }
+ } catch (KeywordSearchModuleException ex) {
+ logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex);
+ //this means Solr is not properly initialized
String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg");
- logger.log(Level.SEVERE, msg);
String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg);
- services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details));
+ services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details));
throw new IngestModuleException(msg);
}
- } catch (KeywordSearchModuleException ex) {
- logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex);
- //this means Solr is not properly initialized
- String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg");
- String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg);
- services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), msg, details));
- throw new IngestModuleException(msg);
- }
- try {
- // make an actual query to verify that server is responding
- // we had cases where getStatus was OK, but the connection resulted in a 404
- server.queryNumIndexedDocuments();
- } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
- throw new IngestModuleException(
- NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg",
- ex.getMessage()));
- }
+ try {
+ // make an actual query to verify that server is responding
+ // we had cases where getStatus was OK, but the connection resulted in a 404
+ server.queryNumIndexedDocuments();
+ } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
+ throw new IngestModuleException(
+ NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg",
+ ex.getMessage()));
+ }
+ // check if this job has any searchable keywords
+ List keywordLists = KeywordSearchListsXML.getCurrent().getListsL();
+ boolean hasKeywordsForSearch = false;
+ for (KeywordList keywordList : keywordLists) {
+ if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) {
+ hasKeywordsForSearch = true;
+ break;
+ }
+ }
+ if (!hasKeywordsForSearch) {
+ services.postMessage(IngestMessage.createWarningMessage(KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"),
+ NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg")));
+ }
+ }
+
//initialize extractors
stringExtractor = new AbstractFileStringExtract(this);
stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts());
@@ -194,57 +191,22 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
//order matters, more specific extractors first
textExtractors.add(new AbstractFileHtmlExtract(this));
textExtractors.add(new AbstractFileTikaTextExtract(this));
-
- ingestStatus = new HashMap<>();
-
- List keywordLists = KeywordSearchListsXML.getCurrent().getListsL();
- boolean hasKeywordsForSearch = false;
- for (KeywordList keywordList : keywordLists) {
- if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) {
- hasKeywordsForSearch = true;
- break;
- }
- }
- if (!hasKeywordsForSearch) {
- services.postMessage(IngestMessage.createWarningMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"),
- NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg")));
- }
-
- processedFiles = false;
- searcherDone = true; //make sure to start the initial currentSearcher
- //keeps track of all results per run not to repeat reporting the same hits
- currentResults = new HashMap<>();
-
- curDataSourceIds = new HashSet<>();
-
+
indexer = new Indexer();
-
- final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000;
- logger.log(Level.INFO, "Using commit interval (ms): {0}", updateIntervalMs);
- logger.log(Level.INFO, "Using searcher interval (ms): {0}", updateIntervalMs);
-
- commitTimer = new Timer(updateIntervalMs, new CommitTimerAction());
- searchTimer = new Timer(updateIntervalMs, new SearchTimerAction());
-
initialized = true;
-
- commitTimer.start();
- searchTimer.start();
}
@Override
public ProcessResult process(AbstractFile abstractFile) {
-
if (initialized == false) //error initializing indexing/Solr
{
logger.log(Level.WARNING, "Skipping processing, module not initialized, file: {0}", abstractFile.getName());
- ingestStatus.put(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
+ putIngestStatus(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
return ProcessResult.OK;
}
try {
//add data source id of the file to the set, keeping track of images being ingested
- final long fileSourceId = caseHandle.getFileDataSource(abstractFile);
- curDataSourceIds.add(fileSourceId);
+ dataSourceId = caseHandle.getFileDataSource(abstractFile);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error getting image id of file processed by keyword search: " + abstractFile.getName(), ex);
@@ -261,14 +223,16 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
return ProcessResult.OK;
}
- processedFiles = true;
-
- //check if it's time to commit after previous processing
- checkRunCommitSearch();
-
//index the file and content (if the content is supported)
indexer.indexFile(abstractFile, true);
+ // Start searching if it hasn't started already
+ if (!startedSearching) {
+ List keywordListNames = settings.getNamesOfEnabledKeyWordLists();
+ SearchRunner.getInstance().startJob(jobId, dataSourceId, keywordListNames);
+ startedSearching = true;
+ }
+
return ProcessResult.OK;
}
@@ -278,39 +242,26 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
*/
@Override
public void shutDown(boolean ingestJobCancelled) {
+ logger.log(Level.INFO, "Instance {0}", instanceNum);
+
if (initialized == false) {
return;
}
if (ingestJobCancelled) {
+ logger.log(Level.INFO, "Ingest job cancelled");
stop();
return;
}
- commitTimer.stop();
-
- //NOTE, we let the 1 before last searcher complete fully, and enqueue the last one
-
- //cancel searcher timer, ensure unwanted searcher does not start
- //before we start the final one
- if (searchTimer.isRunning()) {
- searchTimer.stop();
+ // Remove from the search list and trigger final commit and final search
+ SearchRunner.getInstance().endJob(jobId);
+
+ // We only need to post the summary msg from the last module per job
+ if (refCounter.decrementAndGet(jobId) == 0) {
+ postIndexSummary();
}
- runSearcher = false;
-
- logger.log(Level.INFO, "Running final index commit and search");
- //final commit
- commit();
-
- postIndexSummary();
-
- //run one last search as there are probably some new files committed
- List keywordLists = settings.getNamesOfEnabledKeyWordLists();
- if (!keywordLists.isEmpty() && processedFiles == true) {
- finalSearcher = new Searcher(keywordLists, true); //final searcher run
- finalSearcher.execute();
- }
-
+
//log number of files / chunks in index
//signal a potential change in number of text_ingested files
try {
@@ -321,8 +272,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
} catch (NoOpenCoreException | KeywordSearchModuleException ex) {
logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files/chunks: ", ex);
}
-
- //cleanup done in final searcher
}
/**
@@ -331,22 +280,8 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
private void stop() {
logger.log(Level.INFO, "stop()");
- //stop timer
- commitTimer.stop();
- //stop currentSearcher
- if (currentSearcher != null) {
- currentSearcher.cancel(true);
- }
-
- //cancel searcher timer, ensure unwanted searcher does not start
- if (searchTimer.isRunning()) {
- searchTimer.stop();
- }
- runSearcher = false;
-
- //commit uncommited files, don't search again
- commit();
-
+ SearchRunner.getInstance().stopJob(jobId);
+
cleanup();
}
@@ -354,16 +289,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
* Common cleanup code when module stops or final searcher completes
*/
private void cleanup() {
- ingestStatus.clear();
- currentResults.clear();
- curDataSourceIds.clear();
- currentSearcher = null;
- //finalSearcher = null; //do not collect, might be finalizing
-
- commitTimer.stop();
- searchTimer.stop();
- commitTimer = null;
- //searchTimer = null; // do not collect, final searcher might still be running, in which case it throws an exception
+ synchronized(ingestStatus) {
+ ingestStatus.clear();
+ }
textExtractors.clear();
textExtractors = null;
@@ -374,19 +302,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
initialized = false;
}
- /**
- * Commits index and notifies listeners of index update
- */
- private void commit() {
- if (initialized) {
- logger.log(Level.INFO, "Commiting index");
- ingester.commit();
- logger.log(Level.INFO, "Index comitted");
- //signal a potential change in number of text_ingested files
- indexChangeNotify();
- }
- }
-
/**
* Posts inbox message with summary of text_ingested files
*/
@@ -397,31 +312,34 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
int error_text = 0;
int error_index = 0;
int error_io = 0;
- for (IngestStatus s : ingestStatus.values()) {
- switch (s) {
- case TEXT_INGESTED:
- ++text_ingested;
- break;
- case METADATA_INGESTED:
- ++metadata_ingested;
- break;
- case STRINGS_INGESTED:
- ++strings_ingested;
- break;
- case SKIPPED_ERROR_TEXTEXTRACT:
- error_text++;
- break;
- case SKIPPED_ERROR_INDEXING:
- error_index++;
- break;
- case SKIPPED_ERROR_IO:
- error_io++;
- break;
- default:
- ;
+
+ synchronized(ingestStatus) {
+ for (IngestStatus s : ingestStatus.values()) {
+ switch (s) {
+ case TEXT_INGESTED:
+ ++text_ingested;
+ break;
+ case METADATA_INGESTED:
+ ++metadata_ingested;
+ break;
+ case STRINGS_INGESTED:
+ ++strings_ingested;
+ break;
+ case SKIPPED_ERROR_TEXTEXTRACT:
+ error_text++;
+ break;
+ case SKIPPED_ERROR_INDEXING:
+ error_index++;
+ break;
+ case SKIPPED_ERROR_IO:
+ error_io++;
+ break;
+ default:
+ ;
+ }
}
}
-
+
StringBuilder msg = new StringBuilder();
msg.append("| ").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.knowFileHeaderLbl")).append(" | ").append(text_ingested).append(" | ");
msg.append("| ").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.fileGenStringsHead")).append(" | ").append(strings_ingested).append(" | ");
@@ -432,7 +350,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
msg.append("
");
String indexStats = msg.toString();
logger.log(Level.INFO, "Keyword Indexing Completed: {0}", indexStats);
- services.postMessage(IngestMessage.createMessage(++messageID, MessageType.INFO, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats));
+ services.postMessage(IngestMessage.createMessage(MessageType.INFO, KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxResultsLbl"), indexStats));
if (error_index > 0) {
MessageNotifyUtil.Notify.error(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrsTitle"),
NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.kwIdxErrMsgFiles", error_index));
@@ -442,73 +360,6 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
}
}
- /**
- * Helper method to notify listeners on index update
- */
- private void indexChangeNotify() {
- //signal a potential change in number of text_ingested files
- try {
- final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles();
- KeywordSearch.fireNumIndexedFilesChange(null, new Integer(numIndexedFiles));
- } catch (NoOpenCoreException | KeywordSearchModuleException ex) {
- logger.log(Level.WARNING, "Error executing Solr query to check number of indexed files: ", ex);
- }
- }
-
- /**
- * Check if time to commit, if so, run commit. Then run search if search
- * timer is also set.
- */
- void checkRunCommitSearch() {
- if (commitIndex) {
- logger.log(Level.INFO, "Commiting index");
- commit();
- commitIndex = false;
-
- //after commit, check if time to run searcher
- //NOTE commit/searcher timings don't need to align
- //in worst case, we will run search next time after commit timer goes off, or at the end of ingest
- if (searcherDone && runSearcher) {
- //start search if previous not running
- List keywordLists = settings.getNamesOfEnabledKeyWordLists();
- if (!keywordLists.isEmpty()) {
- currentSearcher = new Searcher(keywordLists);
- currentSearcher.execute();//searcher will stop timer and restart timer when done
- }
- }
- }
- }
-
- /**
- * CommitTimerAction to run by commitTimer Sets a flag to indicate we are
- * ready for commit
- */
- private class CommitTimerAction implements ActionListener {
-
- private final Logger logger = Logger.getLogger(CommitTimerAction.class.getName());
-
- @Override
- public void actionPerformed(ActionEvent e) {
- commitIndex = true;
- logger.log(Level.INFO, "CommitTimer awake");
- }
- }
-
- /**
- * SearchTimerAction to run by searchTimer Sets a flag to indicate we are
- * ready to search
- */
- private class SearchTimerAction implements ActionListener {
-
- private final Logger logger = Logger.getLogger(SearchTimerAction.class.getName());
-
- @Override
- public void actionPerformed(ActionEvent e) {
- runSearcher = true;
- logger.log(Level.INFO, "SearchTimer awake");
- }
- }
-
/**
* File indexer, processes and indexes known/allocated files,
* unknown/unallocated files and directories accordingly
@@ -560,16 +411,16 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
private boolean extractStringsAndIndex(AbstractFile aFile) {
try {
if (stringExtractor.index(aFile)) {
- ingestStatus.put(aFile.getId(), IngestStatus.STRINGS_INGESTED);
+ putIngestStatus(aFile.getId(), IngestStatus.STRINGS_INGESTED);
return true;
} else {
logger.log(Level.WARNING, "Failed to extract strings and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()});
- ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
+ putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
return false;
}
} catch (IngesterException ex) {
logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").", ex);
- ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
+ putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
return false;
}
}
@@ -615,9 +466,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
if ((indexContent == false || aFile.isDir() || size == 0)) {
try {
ingester.ingest(aFile, false); //meta-data only
- ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED);
+ putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED);
} catch (IngesterException ex) {
- ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
+ putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex);
}
return;
@@ -651,9 +502,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) {
try {
ingester.ingest(aFile, false); //meta-data only
- ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED);
+ putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED);
} catch (IngesterException ex) {
- ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
+ putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex);
}
return;
@@ -666,20 +517,20 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
//logger.log(Level.INFO, "indexing: " + aFile.getName());
if (!extractTextAndIndex(aFile, detectedFormat)) {
logger.log(Level.WARNING, "Failed to extract text and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()});
- ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
+ putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
} else {
- ingestStatus.put(aFile.getId(), IngestStatus.TEXT_INGESTED);
+ putIngestStatus(aFile.getId(), IngestStatus.TEXT_INGESTED);
wasTextAdded = true;
}
} catch (IngesterException e) {
logger.log(Level.INFO, "Could not extract text with Tika, " + aFile.getId() + ", "
+ aFile.getName(), e);
- ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
+ putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
} catch (Exception e) {
logger.log(Level.WARNING, "Error extracting text with Tika, " + aFile.getId() + ", "
+ aFile.getName(), e);
- ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
+ putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
}
}
@@ -689,421 +540,4 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
}
}
}
-
- /**
- * Searcher responsible for searching the current index and writing results
- * to blackboard and the inbox. Also, posts results to listeners as Ingest
- * data events. Searches entire index, and keeps track of only new results
- * to report and save. Runs as a background thread.
- */
- private final class Searcher extends SwingWorker {
-
- /**
- * Searcher has private copies/snapshots of the lists and keywords
- */
- private List keywords; //keywords to search
- private List keywordLists; // lists currently being searched
- private Map keywordToList; //keyword to list name mapping
- private AggregateProgressHandle progressGroup;
- private final Logger logger = Logger.getLogger(Searcher.class.getName());
- private boolean finalRun = false;
-
- Searcher(List keywordLists) {
- this.keywordLists = new ArrayList<>(keywordLists);
- this.keywords = new ArrayList<>();
- this.keywordToList = new HashMap<>();
- //keywords are populated as searcher runs
- }
-
- Searcher(List keywordLists, boolean finalRun) {
- this(keywordLists);
- this.finalRun = finalRun;
- }
-
- @Override
- protected Object doInBackground() throws Exception {
- if (finalRun) {
- logger.log(Level.INFO, "Pending start of new (final) searcher");
- } else {
- logger.log(Level.INFO, "Pending start of new searcher");
- }
-
- final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName")
- + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : "");
- progressGroup = AggregateProgressFactory.createSystemHandle(displayName + (" ("
- + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.pendingMsg") + ")"), null, new Cancellable() {
- @Override
- public boolean cancel() {
- logger.log(Level.INFO, "Cancelling the searcher by user.");
- if (progressGroup != null) {
- progressGroup.setDisplayName(displayName + " (" + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.cancelMsg") + "...)");
- }
- return Searcher.this.cancel(true);
- }
- }, null);
-
- updateKeywords();
-
- ProgressContributor[] subProgresses = new ProgressContributor[keywords.size()];
- int i = 0;
- for (Keyword keywordQuery : keywords) {
- subProgresses[i] =
- AggregateProgressFactory.createProgressContributor(keywordQuery.getQuery());
- progressGroup.addContributor(subProgresses[i]);
- i++;
- }
-
- progressGroup.start();
-
- //block to ensure previous searcher is completely done with doInBackground()
- //even after previous searcher cancellation, we need to check this
- searcherLock.lock();
- final StopWatch stopWatch = new StopWatch();
- stopWatch.start();
- try {
- logger.log(Level.INFO, "Started a new searcher");
- progressGroup.setDisplayName(displayName);
- //make sure other searchers are not spawned
- searcherDone = false;
- runSearcher = false;
- if (searchTimer.isRunning()) {
- searchTimer.stop();
- }
-
- int keywordsSearched = 0;
-
- //updateKeywords();
-
- for (Keyword keywordQuery : keywords) {
- if (this.isCancelled()) {
- logger.log(Level.INFO, "Cancel detected, bailing before new keyword processed: {0}", keywordQuery.getQuery());
- return null;
- }
-
- final String queryStr = keywordQuery.getQuery();
- final KeywordList list = keywordToList.get(queryStr);
- final String listName = list.getName();
-
- //new subProgress will be active after the initial query
- //when we know number of hits to start() with
- if (keywordsSearched > 0) {
- subProgresses[keywordsSearched - 1].finish();
- }
-
-
- KeywordSearchQuery del = null;
-
- boolean isRegex = !keywordQuery.isLiteral();
- if (isRegex) {
- del = new TermComponentQuery(keywordQuery);
- } else {
- del = new LuceneQuery(keywordQuery);
- del.escape();
- }
-
- //limit search to currently ingested data sources
- //set up a filter with 1 or more image ids OR'ed
- final KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, curDataSourceIds);
- del.addFilter(dataSourceFilter);
-
- Map> queryResult;
-
- try {
- queryResult = del.performQuery();
- } catch (NoOpenCoreException ex) {
- logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex);
- //no reason to continue with next query if recovery failed
- //or wait for recovery to kick in and run again later
- //likely case has closed and threads are being interrupted
- return null;
- } catch (CancellationException e) {
- logger.log(Level.INFO, "Cancel detected, bailing during keyword query: {0}", keywordQuery.getQuery());
- return null;
- } catch (Exception e) {
- logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e);
- continue;
- }
-
- // calculate new results but substracting results already obtained in this ingest
- // this creates a map of each keyword to the list of unique files that have that hit.
- Map> newResults = filterResults(queryResult, isRegex);
-
- if (!newResults.isEmpty()) {
-
- //write results to BB
-
- //new artifacts created, to report to listeners
- Collection newArtifacts = new ArrayList<>();
-
- //scale progress bar more more granular, per result sub-progress, within per keyword
- int totalUnits = newResults.size();
- subProgresses[keywordsSearched].start(totalUnits);
- int unitProgress = 0;
- String queryDisplayStr = keywordQuery.getQuery();
- if (queryDisplayStr.length() > 50) {
- queryDisplayStr = queryDisplayStr.substring(0, 49) + "...";
- }
- subProgresses[keywordsSearched].progress(listName + ": " + queryDisplayStr, unitProgress);
-
-
- /* cycle through the keywords returned -- only one unless it was a regexp */
- for (final Keyword hitTerm : newResults.keySet()) {
- //checking for cancellation between results
- if (this.isCancelled()) {
- logger.log(Level.INFO, "Cancel detected, bailing before new hit processed for query: {0}", keywordQuery.getQuery());
- return null;
- }
-
- // update progress display
- String hitDisplayStr = hitTerm.getQuery();
- if (hitDisplayStr.length() > 50) {
- hitDisplayStr = hitDisplayStr.substring(0, 49) + "...";
- }
- subProgresses[keywordsSearched].progress(listName + ": " + hitDisplayStr, unitProgress);
- //subProgresses[keywordsSearched].progress(unitProgress);
-
- // this returns the unique files in the set with the first chunk that has a hit
- Map contentHitsFlattened = ContentHit.flattenResults(newResults.get(hitTerm));
- for (final AbstractFile hitFile : contentHitsFlattened.keySet()) {
-
- // get the snippet for the first hit in the file
- String snippet;
- final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery());
- int chunkId = contentHitsFlattened.get(hitFile);
- try {
- snippet = LuceneQuery.querySnippet(snippetQuery, hitFile.getId(), chunkId, isRegex, true);
- } catch (NoOpenCoreException e) {
- logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e);
- //no reason to continue
- return null;
- } catch (Exception e) {
- logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e);
- continue;
- }
-
- // write the blackboard artifact for this keyword in this file
- KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName);
- if (written == null) {
- logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hitFile, hitTerm.toString()});
- continue;
- }
-
- newArtifacts.add(written.getArtifact());
-
- //generate an ingest inbox message for this keyword in this file
- if (list.getIngestMessages()) {
- StringBuilder subjectSb = new StringBuilder();
- StringBuilder detailsSb = new StringBuilder();
- //final int hitFiles = newResults.size();
-
- if (!keywordQuery.isLiteral()) {
- subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExpHitLbl"));
- } else {
- subjectSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLbl"));
- }
- //subjectSb.append("<");
- String uniqueKey = null;
- BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID());
- if (attr != null) {
- final String keyword = attr.getValueString();
- subjectSb.append(keyword);
- uniqueKey = keyword.toLowerCase();
- }
-
- //subjectSb.append(">");
- //String uniqueKey = queryStr;
-
- //details
- detailsSb.append("");
- //hit
- detailsSb.append("");
- detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.kwHitLThLbl"));
- detailsSb.append("| ").append(EscapeUtil.escapeHtml(attr.getValueString())).append(" | ");
- detailsSb.append(" ");
-
- //preview
- attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID());
- if (attr != null) {
- detailsSb.append("");
- detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.previewThLbl"));
- detailsSb.append("| ").append(EscapeUtil.escapeHtml(attr.getValueString())).append(" | ");
- detailsSb.append(" ");
-
- }
-
- //file
- detailsSb.append("");
- detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.fileThLbl"));
- detailsSb.append("| ").append(hitFile.getParentPath()).append(hitFile.getName()).append(" | ");
-
- detailsSb.append(" ");
-
-
- //list
- attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
- detailsSb.append("");
- detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.listThLbl"));
- detailsSb.append("| ").append(attr.getValueString()).append(" | ");
- detailsSb.append(" ");
-
- //regex
- if (!keywordQuery.isLiteral()) {
- attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID());
- if (attr != null) {
- detailsSb.append("");
- detailsSb.append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.regExThLbl"));
- detailsSb.append("| ").append(attr.getValueString()).append(" | ");
- detailsSb.append(" ");
-
- }
- }
- detailsSb.append(" ");
-
- services.postMessage(IngestMessage.createDataMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact()));
- }
- } //for each file hit
-
- ++unitProgress;
-
- }//for each hit term
-
- //update artifact browser
- if (!newArtifacts.isEmpty()) {
- services.fireModuleDataEvent(new ModuleDataEvent(KeywordSearchModuleFactory.getModuleName(), ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts));
- }
- } //if has results
-
- //reset the status text before it goes away
- subProgresses[keywordsSearched].progress("");
-
- ++keywordsSearched;
-
- } //for each keyword
-
- } //end try block
- catch (Exception ex) {
- logger.log(Level.WARNING, "searcher exception occurred", ex);
- } finally {
- try {
- finalizeSearcher();
- stopWatch.stop();
- logger.log(Level.INFO, "Searcher took to run: {0} secs.", stopWatch.getElapsedTimeSecs());
- } finally {
- searcherLock.unlock();
- }
- }
-
- return null;
- }
-
- @Override
- protected void done() {
- // call get to see if there were any errors
- try {
- get();
- } catch (InterruptedException | ExecutionException e) {
- logger.log(Level.SEVERE, "Error performing keyword search: " + e.getMessage());
- services.postMessage(IngestMessage.createErrorMessage(++messageID, KeywordSearchModuleFactory.getModuleName(), "Error performing keyword search", e.getMessage()));
- } // catch and ignore if we were cancelled
- catch (java.util.concurrent.CancellationException ex) {
- }
- }
-
- /**
- * Sync-up the updated keywords from the currently used lists in the XML
- */
- private void updateKeywords() {
- KeywordSearchListsXML loader = KeywordSearchListsXML.getCurrent();
-
- this.keywords.clear();
- this.keywordToList.clear();
-
- for (String name : this.keywordLists) {
- KeywordList list = loader.getList(name);
- for (Keyword k : list.getKeywords()) {
- this.keywords.add(k);
- this.keywordToList.put(k.getQuery(), list);
- }
- }
-
-
- }
-
- //perform all essential cleanup that needs to be done right AFTER doInBackground() returns
- //without relying on done() method that is not guaranteed to run after background thread completes
- //NEED to call this method always right before doInBackground() returns
- /**
- * Performs the cleanup that needs to be done right AFTER
- * doInBackground() returns without relying on done() method that is not
- * guaranteed to run after background thread completes REQUIRED to call
- * this method always right before doInBackground() returns
- */
- private void finalizeSearcher() {
- logger.log(Level.INFO, "Searcher finalizing");
- SwingUtilities.invokeLater(new Runnable() {
- @Override
- public void run() {
- progressGroup.finish();
- }
- });
- searcherDone = true; //next currentSearcher can start
-
- if (finalRun) {
- //this is the final searcher
- logger.log(Level.INFO, "The final searcher in this ingest done.");
-
- //run module cleanup
- cleanup();
- } else {
- //start counting time for a new searcher to start
- //unless final searcher is pending
- if (finalSearcher == null) {
- //we need a new Timer object, because restarting previus will not cause firing of the action
- final int updateIntervalMs = KeywordSearchSettings.getUpdateFrequency().getTime() * 60 * 1000;
- searchTimer = new Timer(updateIntervalMs, new SearchTimerAction());
- searchTimer.start();
- }
- }
- }
-
- //calculate new results but substracting results already obtained in this ingest
- //update currentResults map with the new results
- private Map> filterResults(Map> queryResult, boolean isRegex) {
- Map> newResults = new HashMap<>();
-
- for (String termResult : queryResult.keySet()) {
- List queryTermResults = queryResult.get(termResult);
-
- //translate to list of IDs that we keep track of
- List queryTermResultsIDs = new ArrayList<>();
- for (ContentHit ch : queryTermResults) {
- queryTermResultsIDs.add(ch.getId());
- }
-
- Keyword termResultK = new Keyword(termResult, !isRegex);
- List curTermResults = currentResults.get(termResultK);
- if (curTermResults == null) {
- currentResults.put(termResultK, queryTermResultsIDs);
- newResults.put(termResultK, queryTermResults);
- } else {
- //some AbstractFile hits already exist for this keyword
- for (ContentHit res : queryTermResults) {
- if (!curTermResults.contains(res.getId())) {
- //add to new results
- List newResultsFs = newResults.get(termResultK);
- if (newResultsFs == null) {
- newResultsFs = new ArrayList<>();
- newResults.put(termResultK, newResultsFs);
- }
- newResultsFs.add(res);
- curTermResults.add(res.getId());
- }
- }
- }
- }
-
- return newResults;
-
- }
- }
}
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java
index c07fd6732f..495d171c77 100755
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java
@@ -31,9 +31,7 @@ final class KeywordSearchJobSettings implements IngestModuleIngestJobSettings {
private final HashSet namesOfEnabledKeywordLists = new HashSet<>();
KeywordSearchJobSettings(List namesOfEnabledKeywordLists) {
- for (String keywordList : namesOfEnabledKeywordLists) {
- this.namesOfEnabledKeywordLists.add(keywordList);
- }
+ this.namesOfEnabledKeywordLists.addAll(namesOfEnabledKeywordLists);
}
boolean isKeywordListEnabled(String keywordListName) {
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form
index cb2629ae11..5576e62923 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsManagementPanel.form
@@ -3,7 +3,7 @@
|