mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-18 18:47:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4365 from millmanorama/merge-develop-into-timeline

Browse Source 
Merge develop into timeline
This commit is contained in:
Richard Cordovano 2018-12-13 14:28:33 -05:00 committed by GitHub
commit 781b318f2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
88 changed files with 2894 additions and 1459 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml
View File

@ -19,11 +19,11 @@ install:
- sh travis_build.sh
script:
- set -e
- echo "building autopsy..." && echo -en 'travis_fold:start:script.build\\r'
- echo "Building Autopsy..." && echo -en 'travis_fold:start:script.build\\r'
- cd $TRAVIS_BUILD_DIR/
- ant -q build
- ant build
- echo -en 'travis_fold:end:script.build\\r'
- echo "testing autopsy..." && echo -en 'travis_fold:start:script.tests\\r'
- echo "Testing Autopsy..." && echo -en 'travis_fold:start:script.tests\\r'
- cd Core/
- xvfb-run ant -q test
- echo -en 'travis_fold:end:script.tests\\r'

5
Core/ivy.xml
View File

@ -35,6 +35,11 @@
<dependency conf="core->default" org="com.fasterxml.jackson.core" name="jackson-core" rev="2.9.7"/>
<dependency conf="core->default" org="commons-validator" name="commons-validator" rev="1.6"/>
<dependency conf="core->default" org="net.htmlparser.jericho" name="jericho-html" rev="3.3"/>
<!-- Tika 1.14 seems to declare a (transitive?) dependency on cleartk-util 3.2.2, but the most recent
version available is 2.0.0 Overriding the version worked-->
<override org="org.cleartk" module="cleartk-util" rev="2.0.0"/>
</dependencies>
</ivy-module>

48
Core/nbproject/project.properties
View File

@ -1,26 +1,59 @@
file.reference.activemq-all-5.11.1.jar=release/modules/ext/activemq-all-5.11.1.jar
file.reference.apache-mime4j-core-0.8.1.jar=release/modules/ext/apache-mime4j-core-0.8.1.jar
file.reference.apache-mime4j-dom-0.8.1.jar=release/modules/ext/apache-mime4j-dom-0.8.1.jar
file.reference.asm-5.0.4.jar=release/modules/ext/asm-5.0.4.jar
file.reference.bcmail-jdk15on-1.54.jar=release/modules/ext/bcmail-jdk15on-1.54.jar
file.reference.bcprov-jdk15on-1.54.jar=release/modules/ext/bcprov-jdk15on-1.54.jar
file.reference.boilerpipe-1.1.0.jar=release/modules/ext/boilerpipe-1.1.0.jar
file.reference.c3p0-0.9.5.jar=release/modules/ext/c3p0-0.9.5.jar
file.reference.cdm-4.5.5.jar=release/modules/ext/cdm-4.5.5.jar
file.reference.commons-codec-1.6.jar=release/modules/ext/commons-codec-1.6.jar
file.reference.commons-compress-1.14.jar=release/modules/ext/commons-compress-1.14.jar
file.reference.commons-dbcp2-2.1.1.jar=release\\modules\\ext\\commons-dbcp2-2.1.1.jar
file.reference.commons-pool2-2.4.2.jar=release\\modules\\ext\\commons-pool2-2.4.2.jar
file.reference.commons-dbcp2-2.1.1.jar=release/modules/ext/commons-dbcp2-2.1.1.jar
file.reference.commons-io-2.5.jar=release/modules/ext/commons-io-2.5.jar
file.reference.commons-pool2-2.4.2.jar=release/modules/ext/commons-pool2-2.4.2.jar
file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar
file.reference.geoapi-3.0.0.jar=release/modules/ext/geoapi-3.0.0.jar
file.reference.grib-4.5.5.jar=release/modules/ext/grib-4.5.5.jar
file.reference.gson-2.8.1.jar=release/modules/ext/gson-2.8.1.jar
file.reference.httpservices-4.5.5.jar=release/modules/ext/httpservices-4.5.5.jar
file.reference.isoparser-1.1.18.jar=release/modules/ext/isoparser-1.1.18.jar
file.reference.jackcess-2.2.0.jar=release/modules/ext/jackcess-2.2.0.jar
file.reference.jackcess-encrypt-2.1.4.jar=release/modules/ext/jackcess-encrypt-2.1.4.jar
file.reference.java-libpst-0.8.1.jar=release/modules/ext/java-libpst-0.8.1.jar
file.reference.jcl-over-slf4j-1.7.24.jar=release/modules/ext/jcl-over-slf4j-1.7.24.jar
file.reference.jackson-core-2.9.7.jar=release/modules/ext/jackson-core-2.9.7.jar
file.reference.jdom-2.0.5-contrib.jar=release/modules/ext/jdom-2.0.5-contrib.jar
file.reference.jdom-2.0.5.jar=release/modules/ext/jdom-2.0.5.jar
file.reference.jericho-html-3.3.jar=release/modules/ext/jericho-html-3.3.jar
file.reference.jgraphx-v3.8.0.jar=release/modules/ext/jgraphx-v3.8.0.jar
file.reference.jhighlight-1.0.2.jar=release/modules/ext/jhighlight-1.0.2.jar
file.reference.jmatio-1.2.jar=release/modules/ext/jmatio-1.2.jar
file.reference.json-1.8.jar=release/modules/ext/json-1.8.jar
file.reference.json-simple-1.1.1.jar=release/modules/ext/json-simple-1.1.1.jar
file.reference.jsoup-1.10.3.jar=release/modules/ext/jsoup-1.10.3.jar
file.reference.jul-to-slf4j-1.7.24.jar=release/modules/ext/jul-to-slf4j-1.7.24.jar
file.reference.juniversalchardet-1.0.3.jar=release/modules/ext/juniversalchardet-1.0.3.jar
file.reference.junrar-0.7.jar=release/modules/ext/junrar-0.7.jar
file.reference.jython-standalone-2.7.0.jar=release/modules/ext/jython-standalone-2.7.0.jar
file.reference.mchange-commons-java-0.2.9.jar=release/modules/ext/mchange-commons-java-0.2.9.jar
file.reference.metadata-extractor-2.10.1.jar=release/modules/ext/metadata-extractor-2.10.1.jar
file.reference.netcdf4-4.5.5.jar=release/modules/ext/netcdf4-4.5.5.jar
file.reference.opennlp-tools-1.8.3.jar=release/modules/ext/opennlp-tools-1.8.3.jar
file.reference.poi-3.17.jar=release/modules/ext/poi-3.17.jar
file.reference.poi-ooxml-3.17.jar=release/modules/ext/poi-ooxml-3.17.jar
file.reference.poi-scratchpad-3.17.jar=release/modules/ext/poi-scratchpad-3.17.jar
file.reference.postgresql-9.4.1211.jre7.jar=release/modules/ext/postgresql-9.4.1211.jre7.jar
file.reference.Rejistry-1.0-SNAPSHOT.jar=release/modules/ext/Rejistry-1.0-SNAPSHOT.jar
file.reference.rome-1.5.1.jar=release/modules/ext/rome-1.5.1.jar
file.reference.sevenzipjbinding-AllPlatforms.jar=release/modules/ext/sevenzipjbinding-AllPlatforms.jar
file.reference.sevenzipjbinding.jar=release/modules/ext/sevenzipjbinding.jar
file.reference.sqlite-jdbc-3.8.11.jar=release\\modules\\ext\\sqlite-jdbc-3.8.11.jar
file.reference.sis-metadata-0.6.jar=release/modules/ext/sis-metadata-0.6.jar
file.reference.sis-netcdf-0.6.jar=release/modules/ext/sis-netcdf-0.6.jar
file.reference.sis-utility-0.6.jar=release/modules/ext/sis-utility-0.6.jar
file.reference.slf4j-api-1.7.24.jar=release/modules/ext/slf4j-api-1.7.24.jar
file.reference.sqlite-jdbc-3.8.11.jar=release/modules/ext/sqlite-jdbc-3.8.11.jar
file.reference.StixLib.jar=release/modules/ext/StixLib.jar
file.reference.bcprov-jdk15on-1.54.jar=release/modules/ext/bcprov-jdk15on-1.54.jar
file.reference.jackcess-2.2.0.jar=release/modules/ext/jackcess-2.2.0.jar
file.reference.jackcess-encrypt-2.1.4.jar=release/modules/ext/jackcess-encrypt-2.1.4.jar
file.reference.jempbox-1.8.13.jar=release/modules/ext/jempbox-1.8.13.jar
file.reference.javax.ws.rs-api-2.0.1.jar=release/modules/ext/javax.ws.rs-api-2.0.1.jar
file.reference.cxf-core-3.0.16.jar=release/modules/ext/cxf-core-3.0.16.jar
@ -31,11 +64,14 @@ file.reference.fontbox-2.0.8.jar=release/modules/ext/fontbox-2.0.8.jar
file.reference.pdfbox-2.0.8.jar=release/modules/ext/pdfbox-2.0.8.jar
file.reference.pdfbox-tools-2.0.8.jar=release/modules/ext/pdfbox-tools-2.0.8.jar
file.reference.sleuthkit-postgresql-4.6.4.jar=release/modules/ext/sleuthkit-postgresql-4.6.4.jar
file.reference.tagsoup-1.2.1.jar=release/modules/ext/tagsoup-1.2.1.jar
file.reference.tika-core-1.17.jar=release/modules/ext/tika-core-1.17.jar
file.reference.tika-parsers-1.17.jar=release/modules/ext/tika-parsers-1.17.jar
file.reference.curator-client-2.8.0.jar=release/modules/ext/curator-client-2.8.0.jar
file.reference.curator-framework-2.8.0.jar=release/modules/ext/curator-framework-2.8.0.jar
file.reference.curator-recipes-2.8.0.jar=release/modules/ext/curator-recipes-2.8.0.jar
file.reference.vorbis-java-core-0.8.jar=release/modules/ext/vorbis-java-core-0.8.jar
file.reference.vorbis-java-tika-0.8.jar=release/modules/ext/vorbis-java-tika-0.8.jar
file.reference.xmpcore-5.1.3.jar=release/modules/ext/xmpcore-5.1.3.jar
file.reference.xz-1.6.jar=release/modules/ext/xz-1.6.jar
file.reference.zookeeper-3.4.6.jar=release/modules/ext/zookeeper-3.4.6.jar

302
Core/nbproject/project.xml
View File

@ -338,81 +338,59 @@
<package>org.sleuthkit.autopsy.modules.vmextractor</package>
<package>org.sleuthkit.autopsy.progress</package>
<package>org.sleuthkit.autopsy.report</package>
<package>org.sleuthkit.autopsy.textextractors</package>
<package>org.sleuthkit.autopsy.textextractors.extractionconfigs</package>
<package>org.sleuthkit.autopsy.texttranslation</package>
<package>org.sleuthkit.datamodel</package>
</public-packages>
<class-path-extension>
<runtime-relative-path>ext/apache-mime4j-dom-0.8.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/apache-mime4j-dom-0.8.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jackcess-2.2.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jackcess-2.2.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/zookeeper-3.4.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/zookeeper-3.4.6.jar</binary-origin>
<runtime-relative-path>ext/jericho-html-3.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jericho-html-3.3.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jdom-2.0.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jdom-2.0.5.jar</binary-origin>
<runtime-relative-path>ext/cdm-4.5.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cdm-4.5.5.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/cxf-rt-transports-http-3.0.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cxf-rt-transports-http-3.0.16.jar</binary-origin>
<runtime-relative-path>ext/httpservices-4.5.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/httpservices-4.5.5.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-validator-1.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-validator-1.6.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/curator-framework-2.8.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/curator-framework-2.8.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/bcprov-jdk15on-1.54.jar</runtime-relative-path>
<binary-origin>release/modules/ext/bcprov-jdk15on-1.54.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-compress-1.14.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-compress-1.14.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/fontbox-2.0.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/fontbox-2.0.8.jar</binary-origin>
<runtime-relative-path>ext/geoapi-3.0.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/geoapi-3.0.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-dbcp2-2.1.1.jar</runtime-relative-path>
<binary-origin>release\modules\ext\commons-dbcp2-2.1.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jgraphx-v3.8.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jgraphx-v3.8.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jython-standalone-2.7.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jython-standalone-2.7.0.jar</binary-origin>
<runtime-relative-path>ext/boilerpipe-1.1.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/boilerpipe-1.1.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sevenzipjbinding.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sevenzipjbinding.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sleuthkit-postgresql-4.6.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sleuthkit-postgresql-4.6.4.jar</binary-origin>
<runtime-relative-path>ext/bcmail-jdk15on-1.54.jar</runtime-relative-path>
<binary-origin>release/modules/ext/bcmail-jdk15on-1.54.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/mchange-commons-java-0.2.9.jar</runtime-relative-path>
<binary-origin>release/modules/ext/mchange-commons-java-0.2.9.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/cxf-core-3.0.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cxf-core-3.0.16.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/javax.ws.rs-api-2.0.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/javax.ws.rs-api-2.0.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/postgresql-9.4.1211.jre7.jar</runtime-relative-path>
<binary-origin>release/modules/ext/postgresql-9.4.1211.jre7.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/curator-recipes-2.8.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/curator-recipes-2.8.0.jar</binary-origin>
@ -421,6 +399,14 @@
<runtime-relative-path>ext/metadata-extractor-2.10.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/metadata-extractor-2.10.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/apache-mime4j-core-0.8.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/apache-mime4j-core-0.8.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/tagsoup-1.2.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/tagsoup-1.2.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/tika-core-1.17.jar</runtime-relative-path>
<binary-origin>release/modules/ext/tika-core-1.17.jar</binary-origin>
@ -429,45 +415,37 @@
<runtime-relative-path>ext/StixLib.jar</runtime-relative-path>
<binary-origin>release/modules/ext/StixLib.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/curator-client-2.8.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/curator-client-2.8.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jackson-core-2.9.7.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jackson-core-2.9.7.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/cxf-rt-frontend-jaxrs-3.0.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cxf-rt-frontend-jaxrs-3.0.16.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/pdfbox-tools-2.0.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/pdfbox-tools-2.0.8.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/asm-5.0.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/asm-5.0.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jcl-over-slf4j-1.7.24.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jcl-over-slf4j-1.7.24.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/tika-parsers-1.17.jar</runtime-relative-path>
<binary-origin>release/modules/ext/tika-parsers-1.17.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sqlite-jdbc-3.8.11.jar</runtime-relative-path>
<binary-origin>release\modules\ext\sqlite-jdbc-3.8.11.jar</binary-origin>
<binary-origin>release/modules/ext/sqlite-jdbc-3.8.11.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/activemq-all-5.11.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/activemq-all-5.11.1.jar</binary-origin>
<runtime-relative-path>ext/json-simple-1.1.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/json-simple-1.1.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/xz-1.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/xz-1.6.jar</binary-origin>
<runtime-relative-path>ext/sis-utility-0.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sis-utility-0.6.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/Rejistry-1.0-SNAPSHOT.jar</runtime-relative-path>
<binary-origin>release/modules/ext/Rejistry-1.0-SNAPSHOT.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/dd-plist-1.20.jar</runtime-relative-path>
<binary-origin>release/modules/ext/dd-plist-1.20.jar</binary-origin>
<runtime-relative-path>ext/jhighlight-1.0.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jhighlight-1.0.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jempbox-1.8.13.jar</runtime-relative-path>
@ -477,21 +455,9 @@
<runtime-relative-path>ext/cxf-rt-rs-client-3.0.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cxf-rt-rs-client-3.0.16.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sevenzipjbinding-AllPlatforms.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sevenzipjbinding-AllPlatforms.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-pool2-2.4.2.jar</runtime-relative-path>
<binary-origin>release\modules\ext\commons-pool2-2.4.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jackcess-encrypt-2.1.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jackcess-encrypt-2.1.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jsoup-1.10.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jsoup-1.10.3.jar</binary-origin>
<binary-origin>release/modules/ext/commons-pool2-2.4.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jdom-2.0.5-contrib.jar</runtime-relative-path>
@ -513,6 +479,190 @@
<runtime-relative-path>ext/xmpcore-5.1.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/xmpcore-5.1.3.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/zookeeper-3.4.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/zookeeper-3.4.6.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jdom-2.0.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jdom-2.0.5.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/cxf-rt-transports-http-3.0.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cxf-rt-transports-http-3.0.16.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sis-metadata-0.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sis-metadata-0.6.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/isoparser-1.1.18.jar</runtime-relative-path>
<binary-origin>release/modules/ext/isoparser-1.1.18.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sleuthkit-postgresql-4.6.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sleuthkit-postgresql-4.6.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/vorbis-java-core-0.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/vorbis-java-core-0.8.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-codec-1.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-codec-1.6.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/netcdf4-4.5.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/netcdf4-4.5.5.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/slf4j-api-1.7.24.jar</runtime-relative-path>
<binary-origin>release/modules/ext/slf4j-api-1.7.24.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/java-libpst-0.8.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/java-libpst-0.8.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jul-to-slf4j-1.7.24.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jul-to-slf4j-1.7.24.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/gson-2.8.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/gson-2.8.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/poi-3.17.jar</runtime-relative-path>
<binary-origin>release/modules/ext/poi-3.17.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/poi-scratchpad-3.17.jar</runtime-relative-path>
<binary-origin>release/modules/ext/poi-scratchpad-3.17.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sis-netcdf-0.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sis-netcdf-0.6.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-io-2.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-io-2.5.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/curator-framework-2.8.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/curator-framework-2.8.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/bcprov-jdk15on-1.54.jar</runtime-relative-path>
<binary-origin>release/modules/ext/bcprov-jdk15on-1.54.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/fontbox-2.0.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/fontbox-2.0.8.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/commons-dbcp2-2.1.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/commons-dbcp2-2.1.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jgraphx-v3.8.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jgraphx-v3.8.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/juniversalchardet-1.0.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/juniversalchardet-1.0.3.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jython-standalone-2.7.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jython-standalone-2.7.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jackcess-encrypt-2.1.4.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jackcess-encrypt-2.1.4.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/cxf-core-3.0.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cxf-core-3.0.16.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/javax.ws.rs-api-2.0.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/javax.ws.rs-api-2.0.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/opennlp-tools-1.8.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/opennlp-tools-1.8.3.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/junrar-0.7.jar</runtime-relative-path>
<binary-origin>release/modules/ext/junrar-0.7.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/postgresql-9.4.1211.jre7.jar</runtime-relative-path>
<binary-origin>release/modules/ext/postgresql-9.4.1211.jre7.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/poi-ooxml-3.17.jar</runtime-relative-path>
<binary-origin>release/modules/ext/poi-ooxml-3.17.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/curator-client-2.8.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/curator-client-2.8.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jackson-core-2.9.7.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jackson-core-2.9.7.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/cxf-rt-frontend-jaxrs-3.0.16.jar</runtime-relative-path>
<binary-origin>release/modules/ext/cxf-rt-frontend-jaxrs-3.0.16.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/grib-4.5.5.jar</runtime-relative-path>
<binary-origin>release/modules/ext/grib-4.5.5.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jackson-core-2.9.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jackson-core-2.9.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/activemq-all-5.11.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/activemq-all-5.11.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/xz-1.6.jar</runtime-relative-path>
<binary-origin>release/modules/ext/xz-1.6.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/Rejistry-1.0-SNAPSHOT.jar</runtime-relative-path>
<binary-origin>release/modules/ext/Rejistry-1.0-SNAPSHOT.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/dd-plist-1.20.jar</runtime-relative-path>
<binary-origin>release/modules/ext/dd-plist-1.20.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/rome-1.5.1.jar</runtime-relative-path>
<binary-origin>release/modules/ext/rome-1.5.1.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sevenzipjbinding-AllPlatforms.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sevenzipjbinding-AllPlatforms.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jmatio-1.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jmatio-1.2.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jsoup-1.10.3.jar</runtime-relative-path>
<binary-origin>release/modules/ext/jsoup-1.10.3.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/vorbis-java-tika-0.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/vorbis-java-tika-0.8.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/json-1.8.jar</runtime-relative-path>
<binary-origin>release/modules/ext/json-1.8.jar</binary-origin>
</class-path-extension>
</data>
</configuration>
</project>

32
Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.form
View File

@ -31,14 +31,6 @@
<EmptySpace min="-2" pref="2" max="-2" attributes="0"/>
</Group>
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="pathLabel" min="-2" max="-2" attributes="0"/>
<Component id="errorLabel" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="noFatOrphansCheckbox" alignment="0" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace min="0" pref="0" max="32767" attributes="0"/>
</Group>
<Group type="102" alignment="1" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="sha256HashLabel" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="sha1HashLabel" alignment="0" min="-2" max="-2" attributes="0"/>
@ -47,14 +39,22 @@
<Component id="timeZoneLabel" alignment="0" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="timeZoneComboBox" min="-2" pref="215" max="-2" attributes="0"/>
<Component id="sectorSizeComboBox" min="-2" pref="85" max="-2" attributes="0"/>
<Component id="md5HashTextField" min="-2" pref="231" max="-2" attributes="0"/>
<Component id="sha1HashTextField" min="-2" pref="287" max="-2" attributes="0"/>
<Component id="sha256HashTextField" min="-2" pref="455" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" max="-2" attributes="0">
<Component id="sha256HashTextField" pref="455" max="32767" attributes="0"/>
<Component id="sha1HashTextField" alignment="0" max="32767" attributes="0"/>
<Component id="md5HashTextField" alignment="0" max="32767" attributes="0"/>
<Component id="sectorSizeComboBox" alignment="0" max="32767" attributes="0"/>
<Component id="timeZoneComboBox" alignment="0" max="32767" attributes="0"/>
</Group>
<EmptySpace pref="11" max="32767" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
</Group>
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="pathLabel" min="-2" max="-2" attributes="0"/>
<Component id="errorLabel" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="noFatOrphansCheckbox" alignment="0" min="-2" pref="262" max="-2" attributes="0"/>
</Group>
<EmptySpace min="0" pref="325" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -96,7 +96,7 @@
</Group>
<EmptySpace type="separate" max="-2" attributes="0"/>
<Component id="errorLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="45" max="32767" attributes="0"/>
<EmptySpace pref="23" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>

26
Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java
View File

@ -205,12 +205,6 @@ public class ImageFilePanel extends JPanel implements DocumentListener {
.addComponent(browseButton)
.addGap(2, 2, 2))
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(pathLabel)
.addComponent(errorLabel)
.addComponent(noFatOrphansCheckbox))
.addGap(0, 0, Short.MAX_VALUE))
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(sha256HashLabel)
.addComponent(sha1HashLabel)
@ -218,13 +212,19 @@ public class ImageFilePanel extends JPanel implements DocumentListener {
.addComponent(sectorSizeLabel)
.addComponent(timeZoneLabel))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
.addComponent(sha256HashTextField, javax.swing.GroupLayout.DEFAULT_SIZE, 455, Short.MAX_VALUE)
.addComponent(sha1HashTextField)
.addComponent(md5HashTextField)
.addComponent(sectorSizeComboBox, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(timeZoneComboBox, 0, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(timeZoneComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 215, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(sectorSizeComboBox, javax.swing.GroupLayout.PREFERRED_SIZE, 85, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(md5HashTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 231, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(sha1HashTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 287, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(sha256HashTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 455, javax.swing.GroupLayout.PREFERRED_SIZE))
.addContainerGap(11, Short.MAX_VALUE))
.addComponent(pathLabel)
.addComponent(errorLabel)
.addComponent(noFatOrphansCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 262, javax.swing.GroupLayout.PREFERRED_SIZE))
.addGap(0, 325, Short.MAX_VALUE))
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@ -258,7 +258,7 @@ public class ImageFilePanel extends JPanel implements DocumentListener {
.addComponent(sha256HashLabel))
.addGap(18, 18, 18)
.addComponent(errorLabel)
.addContainerGap(45, Short.MAX_VALUE))
.addContainerGap(23, Short.MAX_VALUE))
);
}// </editor-fold>//GEN-END:initComponents

18
Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/OtherOccurrenceNodeInstanceData.java
View File

@ -122,20 +122,8 @@ class OtherOccurrenceNodeInstanceData implements OtherOccurrenceNodeData {
*/
boolean isCentralRepoNode() {
return (originalCorrelationInstance != null);
}
/**
* Uses the saved instance plus type and value to make a new CorrelationAttribute.
* Should only be called if isCentralRepoNode() is true.
* @return the newly created CorrelationAttribute
*/
CorrelationAttributeInstance getCorrelationAttribute() throws EamDbException {
if (! isCentralRepoNode() ) {
throw new EamDbException("Can not create CorrelationAttribute for non central repo node");
}
return originalCorrelationInstance;
}
}
/**
* Get the case name
* @return the case name
@ -206,7 +194,7 @@ class OtherOccurrenceNodeInstanceData implements OtherOccurrenceNodeData {
* @return the original abstract file
*/
AbstractFile getAbstractFile() throws EamDbException {
if (originalCorrelationInstance == null) {
if (originalAbstractFile == null) {
throw new EamDbException("AbstractFile is null");
}
return originalAbstractFile;

82
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java
View File

@ -58,8 +58,8 @@ abstract class AbstractSqlEamDb implements EamDb {
private final static Logger logger = Logger.getLogger(AbstractSqlEamDb.class.getName());
static final String SCHEMA_MAJOR_VERSION_KEY = "SCHEMA_VERSION";
static final String SCHEMA_MINOR_VERSION_KEY = "SCHEMA_MINOR_VERSION";
static final String CREATED_SCHEMA_MAJOR_VERSION_KEY = "CREATED_SCHEMA_MAJOR_VERSION";
static final String CREATED_SCHEMA_MINOR_VERSION_KEY = "CREATED_SCHEMA_MINOR_VERSION";
static final String CREATION_SCHEMA_MAJOR_VERSION_KEY = "CREATION_SCHEMA_MAJOR_VERSION";
static final String CREATION_SCHEMA_MINOR_VERSION_KEY = "CREATION_SCHEMA_MINOR_VERSION";
static final CaseDbSchemaVersionNumber CURRENT_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 2);
protected final List<CorrelationAttributeInstance.Type> defaultCorrelationTypes;
@ -3184,27 +3184,41 @@ abstract class AbstractSqlEamDb implements EamDb {
statement = conn.createStatement();
int minorVersion = 0;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='SCHEMA_MINOR_VERSION'");
String minorVersionStr = null;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "'");
if (resultSet.next()) {
String minorVersionStr = resultSet.getString("value");
minorVersionStr = resultSet.getString("value");
try {
minorVersion = Integer.parseInt(minorVersionStr);
} catch (NumberFormatException ex) {
throw new EamDbException("Bad value for schema minor version (" + minorVersionStr + ") - database is corrupt", ex);
}
} else {
throw new EamDbException("Failed to read schema minor version from db_info table");
}
int majorVersion = 0;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='SCHEMA_VERSION'");
String majorVersionStr = null;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='" + AbstractSqlEamDb.SCHEMA_MAJOR_VERSION_KEY + "'");
if (resultSet.next()) {
String majorVersionStr = resultSet.getString("value");
majorVersionStr = resultSet.getString("value");
try {
majorVersion = Integer.parseInt(majorVersionStr);
} catch (NumberFormatException ex) {
throw new EamDbException("Bad value for schema version (" + majorVersionStr + ") - database is corrupt", ex);
}
} else {
throw new EamDbException("Failed to read schema major version from db_info table");
}
/*
* IMPORTANT: The code that follows had a bug in it prior to Autopsy
* 4.10.0. The consequence of the bug is that the schema version
* number is always reset to 1.0 or 1.1 if a Central Repository is
* opened by an Autopsy 4.9.1 or earlier client. To cope with this,
* there is an effort in updates to 1.2 and greater to not retry
* schema updates that may already have been done once.
*/
CaseDbSchemaVersionNumber dbSchemaVersion = new CaseDbSchemaVersionNumber(majorVersion, minorVersion);
if (dbSchemaVersion.equals(CURRENT_DB_SCHEMA_VERSION)) {
logger.log(Level.INFO, "Central Repository is up to date");
@ -3215,7 +3229,11 @@ abstract class AbstractSqlEamDb implements EamDb {
return;
}
// Update from 1.0 to 1.1
EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform();
/*
* Update to 1.1
*/
if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 1)) < 0) {
statement.execute("ALTER TABLE reference_sets ADD COLUMN known_status INTEGER;"); //NON-NLS
statement.execute("ALTER TABLE reference_sets ADD COLUMN read_only BOOLEAN;"); //NON-NLS
@ -3226,9 +3244,11 @@ abstract class AbstractSqlEamDb implements EamDb {
// regardless of whether this succeeds.
EamDbUtil.insertDefaultOrganization(conn);
}
//Update to 1.2
/*
* Update to 1.2
*/
if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) {
EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform();
final String addIntegerColumnTemplate = "ALTER TABLE %s ADD COLUMN %s INTEGER;"; //NON-NLS
final String addSsidTableTemplate;
final String addCaseIdIndexTemplate;
@ -3362,26 +3382,50 @@ abstract class AbstractSqlEamDb implements EamDb {
}
/*
* Put values into the db_info table indicating that the
* creation schema version is not known.
* Drop the db_info table and add it back in with the name
* column having a UNIQUE constraint. The name column could now
* be used as the primary key, but the essentially useless id
* column is retained for the sake of backwards compatibility.
* Note that the creation schema version number is set to 0.0
* to indicate that it is unknown.
*/
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATED_SCHEMA_MAJOR_VERSION_KEY + "', '0')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATED_SCHEMA_MINOR_VERSION_KEY + "', '0')");
}
if (!updateSchemaVersion(conn)) {
throw new EamDbException("Error updating schema version");
String creationMajorVer;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name = '" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "'");
if (resultSet.next()) {
creationMajorVer = resultSet.getString("value");
} else {
creationMajorVer = "0";
}
String creationMinorVer;
resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name = '" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "'");
if (resultSet.next()) {
creationMinorVer = resultSet.getString("value");
} else {
creationMinorVer = "0";
}
statement.execute("DROP TABLE db_info");
if (selectedPlatform == EamDbPlatformEnum.POSTGRESQL) {
statement.execute("CREATE TABLE db_info (id SERIAL, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)");
} else {
statement.execute("CREATE TABLE db_info (id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)");
}
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.SCHEMA_MAJOR_VERSION_KEY + "','" + majorVersionStr + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "','" + minorVersionStr + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "','" + creationMajorVer + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "','" + creationMinorVer + "')");
}
updateSchemaVersion(conn);
conn.commit();
logger.log(Level.INFO, "Central Repository upgraded to version " + CURRENT_DB_SCHEMA_VERSION);
logger.log(Level.INFO, String.format("Central Repository schema updated to version %s", CURRENT_DB_SCHEMA_VERSION));
} catch (SQLException | EamDbException ex) {
try {
if (conn != null) {
conn.rollback();
}
} catch (SQLException ex2) {
logger.log(Level.SEVERE, "Database rollback failed", ex2);
logger.log(Level.SEVERE, String.format("Central Repository rollback of failed schema update to %s failed", CURRENT_DB_SCHEMA_VERSION), ex2);
}
throw ex;
} finally {

0
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java Normal file → Executable file
View File

67
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java
View File

@ -126,42 +126,17 @@ public class EamDbUtil {
}
/**
* Store the schema version into the db_info table.
*
* This should be called immediately following the database schema being
* loaded.
* Writes the current schema version into the database.
*
* @param conn Open connection to use.
*
* @return true on success, else false
* @throws SQLException If there is an error doing the update.
*/
static boolean updateSchemaVersion(Connection conn) {
Statement statement;
ResultSet resultSet;
try {
statement = conn.createStatement();
resultSet = statement.executeQuery("SELECT id FROM db_info WHERE name='SCHEMA_VERSION'");
if (resultSet.next()) {
int id = resultSet.getInt("id");
statement.execute("UPDATE db_info SET value=" + CURRENT_DB_SCHEMA_VERSION.getMajor() + " WHERE id=" + id);
} else {
statement.execute("INSERT INTO db_info (name, value) VALUES ('SCHEMA_VERSION', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
}
resultSet = statement.executeQuery("SELECT id FROM db_info WHERE name='SCHEMA_MINOR_VERSION'");
if (resultSet.next()) {
int id = resultSet.getInt("id");
statement.execute("UPDATE db_info SET value=" + CURRENT_DB_SCHEMA_VERSION.getMinor() + " WHERE id=" + id);
} else {
statement.execute("INSERT INTO db_info (name, value) VALUES ('SCHEMA_MINOR_VERSION', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
}
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "Error adding schema version to db_info.", ex);
return false;
static void updateSchemaVersion(Connection conn) throws SQLException {
try (Statement statement = conn.createStatement()) {
statement.execute("UPDATE db_info SET value = '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "' WHERE name = '" + AbstractSqlEamDb.SCHEMA_MAJOR_VERSION_KEY + "'");
statement.execute("UPDATE db_info SET value = '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "' WHERE name = '" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "'");
}
return true;
}
/**
@ -191,9 +166,9 @@ public class EamDbUtil {
}
/**
* Upgrade the current central reposity to the newest version. If the upgrade
* fails, the central repository will be disabled and the current settings
* will be cleared.
* Upgrade the current Central Reposity schema to the newest version. If the
* upgrade fails, the Central Repository will be disabled and the current
* settings will be cleared.
*
* @return true if the upgrade succeeds, false otherwise.
*/
@ -201,7 +176,7 @@ public class EamDbUtil {
if (!EamDb.isEnabled()) {
return true;
}
CoordinationService.Lock lock = null;
try {
EamDb db = EamDb.getInstance();
@ -217,23 +192,23 @@ public class EamDbUtil {
LOGGER.log(Level.SEVERE, "Error updating central repository", ex);
// Disable the central repo and clear the current settings.
try{
try {
if (null != EamDb.getInstance()) {
EamDb.getInstance().shutdownConnections();
}
} catch (EamDbException ex2){
} catch (EamDbException ex2) {
LOGGER.log(Level.SEVERE, "Error shutting down central repo connection pool", ex);
}
}
setUseCentralRepo(false);
EamDbPlatformEnum.setSelectedPlatform(EamDbPlatformEnum.DISABLED.name());
EamDbPlatformEnum.saveSelectedPlatform();
return false;
} finally {
if(lock != null){
try{
if (lock != null) {
try {
lock.release();
} catch (CoordinationServiceException ex){
} catch (CoordinationServiceException ex) {
LOGGER.log(Level.SEVERE, "Error releasing database lock", ex);
}
}
@ -254,6 +229,7 @@ public class EamDbUtil {
* Check whether the given org is the default organization.
*
* @param org
*
* @return true if it is the default org, false otherwise
*/
public static boolean isDefaultOrg(EamOrganization org) {
@ -264,6 +240,7 @@ public class EamDbUtil {
* Add the default organization to the database
*
* @param conn
*
* @return true if successful, false otherwise
*/
static boolean insertDefaultOrganization(Connection conn) {
@ -294,7 +271,7 @@ public class EamDbUtil {
* If the Central Repos use has been enabled.
*
* @return true if the Central Repo may be configured, false if it should
* not be able to be
* not be able to be
*/
public static boolean useCentralRepo() {
return Boolean.parseBoolean(ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY));
@ -305,7 +282,7 @@ public class EamDbUtil {
* configured.
*
* @param centralRepoCheckBoxIsSelected - true if the central repo can be
* used
* used
*/
public static void setUseCentralRepo(boolean centralRepoCheckBoxIsSelected) {
ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY, Boolean.toString(centralRepoCheckBoxIsSelected));
@ -364,7 +341,7 @@ public class EamDbUtil {
* Close the prepared statement.
*
* @param preparedStatement The prepared statement to be closed.
*
*
* @deprecated Use closeStatement() instead.
*
* @throws EamDbException

0
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java Normal file → Executable file
View File

18
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java
View File

@ -406,13 +406,6 @@ public final class PostgresEamDbSettings {
String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate();
String instancesObjectIdIdx = getAddObjectIdIndexTemplate();
StringBuilder createDbInfoTable = new StringBuilder();
createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info (");
createDbInfoTable.append("id SERIAL PRIMARY KEY NOT NULL,");
createDbInfoTable.append("name text NOT NULL,");
createDbInfoTable.append("value text NOT NULL");
createDbInfoTable.append(")");
// NOTE: the db_info table currenly only has 1 row, so having an index
// provides no benefit.
Connection conn = null;
@ -438,11 +431,16 @@ public final class PostgresEamDbSettings {
stmt.execute(createCorrelationTypesTable.toString());
stmt.execute(createDbInfoTable.toString());
/*
* Note that the essentially useless id column in the following
* table is required for backwards compatibility. Otherwise, the
* name column could be the primary key.
*/
stmt.execute("CREATE TABLE db_info (id SERIAL, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.SCHEMA_MAJOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATED_SCHEMA_MAJOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATED_SCHEMA_MINOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
// Create a separate instance and reference table for each correlation type
List<CorrelationAttributeInstance.Type> DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes();

26
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java
View File

@ -291,7 +291,7 @@ public final class SqliteEamDbSettings {
createDataSourcesTable.append("datasource_obj_id integer,");
createDataSourcesTable.append("md5 text DEFAULT NULL,");
createDataSourcesTable.append("sha1 text DEFAULT NULL,");
createDataSourcesTable.append("sha256 text DEFAULT NULL,");
createDataSourcesTable.append("sha256 text DEFAULT NULL,");
createDataSourcesTable.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,");
createDataSourcesTable.append("CONSTRAINT datasource_unique UNIQUE (case_id, device_id, name)");
createDataSourcesTable.append(")");
@ -348,13 +348,6 @@ public final class SqliteEamDbSettings {
String instancesValueIdx = getAddValueIndexTemplate();
String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate();
String instancesObjectIdIdx = getAddObjectIdIndexTemplate();
StringBuilder createDbInfoTable = new StringBuilder();
createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info (");
createDbInfoTable.append("id integer primary key NOT NULL,");
createDbInfoTable.append("name text NOT NULL,");
createDbInfoTable.append("value text NOT NULL");
createDbInfoTable.append(")");
// NOTE: the db_info table currenly only has 1 row, so having an index
// provides no benefit.
@ -387,11 +380,16 @@ public final class SqliteEamDbSettings {
stmt.execute(createCorrelationTypesTable.toString());
stmt.execute(createDbInfoTable.toString());
/*
* Note that the essentially useless id column in the following
* table is required for backwards compatibility. Otherwise, the
* name column could be the primary key.
*/
stmt.execute("CREATE TABLE db_info (id INTEGER PRIMARY KEY, name TEXT UNIQUE NOT NULL, value TEXT NOT NULL)");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.SCHEMA_MAJOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.SCHEMA_MINOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATED_SCHEMA_MAJOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATED_SCHEMA_MINOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MAJOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
stmt.execute("INSERT INTO db_info (name, value) VALUES ('" + AbstractSqlEamDb.CREATION_SCHEMA_MINOR_VERSION_KEY + "', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
// Create a separate instance and reference table for each artifact type
List<CorrelationAttributeInstance.Type> DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes();
@ -427,7 +425,7 @@ public final class SqliteEamDbSettings {
}
return true;
}
/**
* Get the template String for creating a new _instances table in a Sqlite
* central repository. %s will exist in the template where the name of the
@ -511,8 +509,8 @@ public final class SqliteEamDbSettings {
* instance table. %s will exist in the template where the name of the new
* table will be addedd.
*
* @return a String which is a template for adding an index to the file_obj_id
* column of a _instances table
* @return a String which is a template for adding an index to the
* file_obj_id column of a _instances table
*/
static String getAddObjectIdIndexTemplate() {
// Each "%s" will be replaced with the relevant TYPE_instances table name.

76
Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java
View File

@ -58,23 +58,24 @@ import org.sleuthkit.datamodel.TskCoreException;
*/
@NbBundle.Messages({"IngestEventsListener.ingestmodule.name=Correlation Engine"})
public class IngestEventsListener {
private static final Logger LOGGER = Logger.getLogger(CorrelationAttributeInstance.class.getName());
private static final String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name();
final Collection<String> recentlyAddedCeArtifacts = new LinkedHashSet<>();
private static int correlationModuleInstanceCount;
private static boolean flagNotableItems;
private static boolean flagSeenDevices;
private static boolean createCrProperties;
private final ExecutorService jobProcessingExecutor;
private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d";
private final PropertyChangeListener pcl1 = new IngestModuleEventListener();
private final PropertyChangeListener pcl2 = new IngestJobEventListener();
IngestEventsListener() {
jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build());
}
void shutdown() {
ThreadUtils.shutDownTaskExecutor(jobProcessingExecutor);
}
@ -149,6 +150,15 @@ public class IngestEventsListener {
return flagSeenDevices;
}
/**
* Are correlation properties being created
*
* @return True if creating correlation properties; otherwise false.
*/
public synchronized static boolean shouldCreateCrProperties() {
return createCrProperties;
}
/**
* Configure the listener to flag notable items or not.
*
@ -166,11 +176,20 @@ public class IngestEventsListener {
public synchronized static void setFlagSeenDevices(boolean value) {
flagSeenDevices = value;
}
/**
* Configure the listener to create correlation properties
*
* @param value True to create properties; otherwise false.
*/
public synchronized static void setCreateCrProperties(boolean value) {
createCrProperties = value;
}
@NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
"IngestEventsListener.prevCaseComment.text=Previous Case: "})
static private void postCorrelatedBadArtifactToBlackboard(BlackboardArtifact bbArtifact, List<String> caseDisplayNames) {
Collection<BlackboardAttribute> attributes = Arrays.asList(
new BlackboardAttribute(
TSK_SET_NAME, MODULE_NAME,
@ -204,17 +223,17 @@ public class IngestEventsListener {
bbArtifact.getArtifactID()));
postArtifactToBlackboard(bbArtifact, attributes);
}
private static void postArtifactToBlackboard(BlackboardArtifact bbArtifact, Collection<BlackboardAttribute> attributes) {
try {
SleuthkitCase tskCase = bbArtifact.getSleuthkitCase();
AbstractFile abstractFile = bbArtifact.getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID());
AbstractFile abstractFile = tskCase.getAbstractFileById(bbArtifact.getObjectID());
Blackboard blackboard = tskCase.getBlackboard();
// Create artifact if it doesn't already exist.
if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_ARTIFACT_HIT, attributes)) {
BlackboardArtifact tifArtifact = abstractFile.newArtifact(TSK_INTERESTING_ARTIFACT_HIT);
tifArtifact.addAttributes(attributes);
try {
// index the artifact for keyword search
blackboard.postArtifact(tifArtifact, MODULE_NAME);
@ -228,9 +247,9 @@ public class IngestEventsListener {
LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
}
}
private class IngestModuleEventListener implements PropertyChangeListener {
@Override
public void propertyChange(PropertyChangeEvent evt) {
//if ingest is running we want there to check if there is a Correlation Engine module running
@ -249,16 +268,17 @@ public class IngestEventsListener {
//if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items
boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems();
boolean flagPrevious = !IngestManager.getInstance().isIngestRunning() || isFlagSeenDevices();
jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable, flagPrevious));
boolean createAttributes = !IngestManager.getInstance().isIngestRunning() || shouldCreateCrProperties();
jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable, flagPrevious, createAttributes));
break;
}
}
}
}
}
private class IngestJobEventListener implements PropertyChangeListener {
@Override
public void propertyChange(PropertyChangeEvent evt) {
switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) {
@ -268,11 +288,11 @@ public class IngestEventsListener {
}
}
}
}
private final class AnalysisCompleteTask implements Runnable {
@Override
public void run() {
// clear the tracker to reduce memory usage
@ -282,21 +302,23 @@ public class IngestEventsListener {
//else another instance of the Correlation Engine Module is still being run.
} // DATA_SOURCE_ANALYSIS_COMPLETED
}
private final class DataAddedTask implements Runnable {
private final EamDb dbManager;
private final PropertyChangeEvent event;
private final boolean flagNotableItemsEnabled;
private final boolean flagPreviousItemsEnabled;
private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled) {
dbManager = db;
event = evt;
private final boolean createCorrelationAttributes;
private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes) {
this.dbManager = db;
this.event = evt;
this.flagNotableItemsEnabled = flagNotableItemsEnabled;
this.flagPreviousItemsEnabled = flagPreviousItemsEnabled;
this.createCorrelationAttributes = createCorrelationAttributes;
}
@Override
public void run() {
if (!EamDb.isEnabled()) {
@ -308,7 +330,7 @@ public class IngestEventsListener {
return;
}
List<CorrelationAttributeInstance> eamArtifacts = new ArrayList<>();
for (BlackboardArtifact bbArtifact : bbArtifacts) {
// eamArtifact will be null OR a EamArtifact containing one EamArtifactInstance.
List<CorrelationAttributeInstance> convertedArtifacts = EamArtifactUtil.makeInstancesFromBlackboardArtifact(bbArtifact, true);
@ -347,7 +369,9 @@ public class IngestEventsListener {
LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
}
}
eamArtifacts.add(eamArtifact);
if (createCorrelationAttributes) {
eamArtifacts.add(eamArtifact);
}
}
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Error counting notable artifacts.", ex);

1
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties
View File

@ -1,3 +1,4 @@
IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings
IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable
IngestSettingsPanel.flagPreviouslySeenDevicesCheckbox.text=Flag previously seen devices
IngestSettingsPanel.createCorrelationPropertiesCheckbox.text=Save items to the Central Repository

73
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModule.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.centralrepository.ingestmodule;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.logging.Level;
@ -48,8 +49,12 @@ import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT;
import org.sleuthkit.datamodel.BlackboardAttribute;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
@ -61,8 +66,11 @@ import org.sleuthkit.datamodel.TskData;
"CentralRepoIngestModule.prevCaseComment.text=Previous Case: "})
final class CentralRepoIngestModule implements FileIngestModule {
private static final String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName();
static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = true;
static final boolean DEFAULT_FLAG_PREVIOUS_DEVICES = true;
static final boolean DEFAULT_CREATE_CR_PROPERTIES = true;
private final static Logger logger = Logger.getLogger(CentralRepoIngestModule.class.getName());
private final IngestServices services = IngestServices.getInstance();
@ -75,6 +83,7 @@ final class CentralRepoIngestModule implements FileIngestModule {
private final boolean flagTaggedNotableItems;
private final boolean flagPreviouslySeenDevices;
private Blackboard blackboard;
private final boolean createCorrelationProperties;
/**
* Instantiate the Correlation Engine ingest module.
@ -84,6 +93,7 @@ final class CentralRepoIngestModule implements FileIngestModule {
CentralRepoIngestModule(IngestSettings settings) {
flagTaggedNotableItems = settings.isFlagTaggedNotableItems();
flagPreviouslySeenDevices = settings.isFlagPreviousDevices();
createCorrelationProperties = settings.shouldCreateCorrelationProperties();
}
@Override
@ -153,27 +163,28 @@ final class CentralRepoIngestModule implements FileIngestModule {
}
}
// insert this file into the central repository
try {
CorrelationAttributeInstance cefi = new CorrelationAttributeInstance(
filesType,
md5,
eamCase,
eamDataSource,
abstractFile.getParentPath() + abstractFile.getName(),
null,
TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
,
abstractFile.getId());
dbManager.addAttributeInstanceBulk(cefi);
} catch (EamDbException ex) {
logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
return ProcessResult.ERROR;
} catch (CorrelationAttributeNormalizationException ex) {
logger.log(Level.INFO, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
return ProcessResult.ERROR;
// insert this file into the central repository
if (createCorrelationProperties) {
try {
CorrelationAttributeInstance cefi = new CorrelationAttributeInstance(
filesType,
md5,
eamCase,
eamDataSource,
abstractFile.getParentPath() + abstractFile.getName(),
null,
TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database.
,
abstractFile.getId());
dbManager.addAttributeInstanceBulk(cefi);
} catch (EamDbException ex) {
logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
return ProcessResult.ERROR;
} catch (CorrelationAttributeNormalizationException ex) {
logger.log(Level.INFO, "Error adding artifact to bulk artifacts.", ex); // NON-NLS
return ProcessResult.ERROR;
}
}
return ProcessResult.OK;
}
@ -236,6 +247,9 @@ final class CentralRepoIngestModule implements FileIngestModule {
if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagSeenDevices()) {
IngestEventsListener.setFlagSeenDevices(flagPreviouslySeenDevices);
}
if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.shouldCreateCrProperties()) {
IngestEventsListener.setCreateCrProperties(createCorrelationProperties);
}
if (EamDb.isEnabled() == false) {
/*
@ -330,17 +344,18 @@ final class CentralRepoIngestModule implements FileIngestModule {
*/
private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames) {
String MODULE_NAME = CentralRepoIngestModuleFactory.getModuleName();
Collection<BlackboardAttribute> attributes = new ArrayList<>();
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME,
Bundle.CentralRepoIngestModule_prevTaggedSet_text()));
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME,
Bundle.CentralRepoIngestModule_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(",", "", ""))));
Collection<BlackboardAttribute> attributes = Arrays.asList(
new BlackboardAttribute(
TSK_SET_NAME, MODULE_NAME,
Bundle.CentralRepoIngestModule_prevTaggedSet_text()),
new BlackboardAttribute(
TSK_COMMENT, MODULE_NAME,
Bundle.CentralRepoIngestModule_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))));
try {
// Create artifact if it doesn't already exist.
if (!blackboard.artifactExists(abstractFile, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, attributes)) {
BlackboardArtifact tifArtifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT);
if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_FILE_HIT, attributes)) {
BlackboardArtifact tifArtifact = abstractFile.newArtifact(TSK_INTERESTING_FILE_HIT);
tifArtifact.addAttributes(attributes);
try {
// index the artifact for keyword search

25
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java
View File

@ -27,8 +27,9 @@ final class IngestSettings implements IngestModuleIngestJobSettings {
private static final long serialVersionUID = 1L;
private boolean flagTaggedNotableItems;
private boolean flagPreviousDevices;
private final boolean flagTaggedNotableItems;
private final boolean flagPreviousDevices;
private final boolean createCorrelationProperties;
/**
* Instantiate the ingest job settings with default values.
@ -36,17 +37,22 @@ final class IngestSettings implements IngestModuleIngestJobSettings {
IngestSettings() {
this.flagTaggedNotableItems = CentralRepoIngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS;
this.flagPreviousDevices = CentralRepoIngestModule.DEFAULT_FLAG_PREVIOUS_DEVICES;
this.createCorrelationProperties = CentralRepoIngestModule.DEFAULT_CREATE_CR_PROPERTIES;
}
/**
* Instantiate the ingest job settings.
*
* @param flagTaggedNotableItems Flag previously tagged notable items.
* @param flagPreviousDevices Flag devices which exist already in the Central Repository
* @param flagTaggedNotableItems Flag previously tagged notable items.
* @param flagPreviousDevices Flag devices which exist already in
* the Central Repository
* @param createCorrelationProperties Create correlation properties in the
* central repository
*/
IngestSettings(boolean flagTaggedNotableItems, boolean flagPreviousDevices) {
IngestSettings(boolean flagTaggedNotableItems, boolean flagPreviousDevices, boolean createCorrelationProperties) {
this.flagTaggedNotableItems = flagTaggedNotableItems;
this.flagPreviousDevices = flagPreviousDevices;
this.createCorrelationProperties = createCorrelationProperties;
}
@Override
@ -71,4 +77,13 @@ final class IngestSettings implements IngestModuleIngestJobSettings {
boolean isFlagPreviousDevices() {
return flagPreviousDevices;
}
/**
* Should correlation properties be created
*
* @return True if creating; otherwise false.
*/
boolean shouldCreateCorrelationProperties() {
return createCorrelationProperties;
}
}

20
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form
View File

@ -22,9 +22,10 @@
<Component id="ingestSettingsLabel" min="-2" max="-2" attributes="0"/>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="10" pref="10" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="flagPreviouslySeenDevicesCheckbox" min="-2" max="-2" attributes="0"/>
<Component id="flagTaggedNotableItemsCheckbox" min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" max="-2" attributes="0">
<Component id="flagTaggedNotableItemsCheckbox" max="32767" attributes="0"/>
<Component id="flagPreviouslySeenDevicesCheckbox" alignment="0" max="32767" attributes="0"/>
<Component id="createCorrelationPropertiesCheckbox" alignment="0" max="32767" attributes="0"/>
</Group>
</Group>
</Group>
@ -37,11 +38,13 @@
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="ingestSettingsLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="9" max="-2" attributes="0"/>
<Component id="createCorrelationPropertiesCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="flagTaggedNotableItemsCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="flagPreviouslySeenDevicesCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="222" max="32767" attributes="0"/>
<EmptySpace pref="197" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -71,5 +74,12 @@
</Property>
</Properties>
</Component>
<Component class="javax.swing.JCheckBox" name="createCorrelationPropertiesCheckbox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties" key="IngestSettingsPanel.createCorrelationPropertiesCheckbox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Form>

20
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java
View File

@ -43,11 +43,12 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel {
private void customizeComponents(IngestSettings settings) {
flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems());
flagPreviouslySeenDevicesCheckbox.setSelected(settings.isFlagPreviousDevices());
createCorrelationPropertiesCheckbox.setSelected(settings.shouldCreateCorrelationProperties());
}
@Override
public IngestModuleIngestJobSettings getSettings() {
return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected(), flagPreviouslySeenDevicesCheckbox.isSelected());
return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected(), flagPreviouslySeenDevicesCheckbox.isSelected(), createCorrelationPropertiesCheckbox.isSelected());
}
/**
@ -62,6 +63,7 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel {
ingestSettingsLabel = new javax.swing.JLabel();
flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox();
flagPreviouslySeenDevicesCheckbox = new javax.swing.JCheckBox();
createCorrelationPropertiesCheckbox = new javax.swing.JCheckBox();
ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N
@ -70,6 +72,8 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel {
org.openide.awt.Mnemonics.setLocalizedText(flagPreviouslySeenDevicesCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagPreviouslySeenDevicesCheckbox.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(createCorrelationPropertiesCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.createCorrelationPropertiesCheckbox.text")); // NOI18N
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
@ -80,9 +84,10 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel {
.addComponent(ingestSettingsLabel)
.addGroup(layout.createSequentialGroup()
.addGap(10, 10, 10)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(flagPreviouslySeenDevicesCheckbox)
.addComponent(flagTaggedNotableItemsCheckbox))))
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
.addComponent(flagTaggedNotableItemsCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(flagPreviouslySeenDevicesCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(createCorrelationPropertiesCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
.addContainerGap(65, Short.MAX_VALUE))
);
layout.setVerticalGroup(
@ -90,15 +95,18 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel {
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(ingestSettingsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGap(9, 9, 9)
.addComponent(createCorrelationPropertiesCheckbox)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(flagTaggedNotableItemsCheckbox)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(flagPreviouslySeenDevicesCheckbox)
.addContainerGap(222, Short.MAX_VALUE))
.addContainerGap(197, Short.MAX_VALUE))
);
}// </editor-fold>//GEN-END:initComponents
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JCheckBox createCorrelationPropertiesCheckbox;
private javax.swing.JCheckBox flagPreviouslySeenDevicesCheckbox;
private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox;
private javax.swing.JLabel ingestSettingsLabel;

64
Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeInstance.java
View File

@ -1,16 +1,16 @@
/*
*
*
* Autopsy Forensic Browser
*
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -47,11 +47,12 @@ public abstract class AbstractCommonAttributeInstance {
* Create a leaf node for attributes found in files in the current case db.
*
* @param abstractFileReference file from which the common attribute was
* found
* @param cachedFiles storage for abstract files which have been used
* already so we can avoid extra roundtrips to the case db
* @param dataSource datasource where this attribute appears
* @param caseName case where this attribute appears
* found
* @param cachedFiles storage for abstract files which have been
* used already so we can avoid extra
* roundtrips to the case db
* @param dataSource datasource where this attribute appears
* @param caseName case where this attribute appears
*/
AbstractCommonAttributeInstance(Long abstractFileReference, String dataSource, String caseName) {
this.abstractFileObjectId = abstractFileReference;
@ -64,17 +65,19 @@ public abstract class AbstractCommonAttributeInstance {
* available in the current data case.
*
* @param cachedFiles storage for abstract files which have been used
* already so we can avoid extra roundtrips to the case db
* already so we can avoid extra roundtrips to the case
* db
*/
AbstractCommonAttributeInstance() {
this.abstractFileObjectId = -1L;
this.caseName = "";
this.dataSource = "";
}
/**
* Get the type of common attribute.
* @return
*
* @return
*/
public abstract CorrelationAttributeInstance.Type getCorrelationAttributeInstanceType();
@ -83,8 +86,8 @@ public abstract class AbstractCommonAttributeInstance {
* CaseDB.
*
* @return AbstractFile corresponding to this common attribute or null if it
* cannot be found (for example, in the event that this is a central repo
* file)
* cannot be found (for example, in the event that this is a central
* repo file)
*/
abstract AbstractFile getAbstractFile();
@ -143,25 +146,38 @@ public abstract class AbstractCommonAttributeInstance {
* Otherwise, we will get an InterCaseCommonAttributeInstanceNode which
* supports only baseline functionality.
*
* @param attributeInstance common file attribute instance form the central
* repo
* @param abstractFile an AbstractFile from which the attribute instance was
* found - applies to CaseDbCommonAttributeInstance only
* @param currentCaseName
* @param attribute common file attribute instance form the central
* repo
* @param abstractFile an AbstractFile from which the attribute instance
* was found - applies to
* CaseDbCommonAttributeInstance only
* @param currentCaseName the name of the current case
* @param nodeType the type of the node to create for determining
* which columns to add
*
* @return the appropriate leaf node for the results tree
*
* @throws TskCoreException
*/
static DisplayableItemNode createNode(CorrelationAttributeInstance attribute, AbstractFile abstractFile, String currentCaseName) throws TskCoreException {
static DisplayableItemNode createNode(CorrelationAttributeInstance attribute, AbstractFile abstractFile, String currentCaseName, NODE_TYPE nodeType) throws TskCoreException {
DisplayableItemNode leafNode;
if (abstractFile == null) {
leafNode = new CentralRepoCommonAttributeInstanceNode(attribute);
leafNode = new CentralRepoCommonAttributeInstanceNode(attribute, nodeType);
} else {
final String abstractFileDataSourceName = abstractFile.getDataSource().getName();
leafNode = new CaseDBCommonAttributeInstanceNode(abstractFile, currentCaseName, abstractFileDataSourceName);
leafNode = new CaseDBCommonAttributeInstanceNode(abstractFile, currentCaseName, abstractFileDataSourceName, attribute.getCorrelationValue(), nodeType);
}
return leafNode;
}
/**
* Enum for type of results being displayed in nodes.
*/
public enum NODE_TYPE {
//depending on the type of results displayed different columns will be necessary to display complete information
CASE_NODE,
COUNT_NODE;
}
}

24
Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java
View File

@ -34,7 +34,7 @@ import org.sleuthkit.datamodel.TskCoreException;
/**
* Prototype for an object which finds files with common attributes. Subclass
* this and implement findMatches in order
* this and implement findMatchesByCount in order
*/
public abstract class AbstractCommonAttributeSearcher {
@ -50,7 +50,7 @@ public abstract class AbstractCommonAttributeSearcher {
/**
* Implement this to search for files with common attributes. Creates an
* object (CommonAttributeSearchResults) which contains all of the
* object (CommonAttributeCountSearchResults) which contains all of the
* information required to display a tree view in the UI. The view will
* contain 3 layers: a top level node, indicating the number matches each of
* it's children possess, a mid level node indicating the matched attribute,
@ -62,7 +62,25 @@ public abstract class AbstractCommonAttributeSearcher {
* @throws SQLException
* @throws EamDbException
*/
public abstract CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException;
public abstract CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException;
/**
* Implement this to search for files with common attributes. Creates an
* object (CommonAttributeCountSearchResults) which contains all of the
* information required to display a tree view in the UI. The view will
* contain 3 layers: a top level node, indicating the name of the case the
* results were found in, a mid level node indicating what data source the
* match was found in, and a bottom level which shows the files the match
* was found in
*
* @return An object containing the results of the search
*
* @throws TskCoreException
* @throws NoCurrentCaseException
* @throws SQLException
* @throws EamDbException
*/
public abstract CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException;
/**
* Implement this to create a descriptive string for the tab which will

14
Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java
View File

@ -52,11 +52,19 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut
}
@Override
public CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
public CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseCommonAttributeValues(Case.getCurrentCase());
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCount(Case.getCurrentCase());
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
}
@Override
public CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findInterCaseValuesByCase(Case.getCurrentCase());
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
}
@NbBundle.Messages({

6
Core/src/org/sleuthkit/autopsy/commonfilesearch/Bundle.properties
View File

@ -40,3 +40,9 @@ CommonAttributePanel.commonItemSearchDescription.text=<html>Find items that exis
CommonAttributePanel.scopeLabel.text=Scope of Search:
InterCasePanel.correlationComboBoxLabel.text=Property Type to Match:
CommonAttributePanel.percentageThresholdInputBox.text=20
CommonAttributePanel.resultsDisplayLabel.text_2=Display results organized by:
CommonAttributePanel.organizeByCaseRadio.text=Case
CommonAttributePanel.organizeByCountRadio.text=Number of occurrences
CommonAttributePanel.caseResultsRadioButton.text=Case
CommonAttributePanel.countResultsRadioButton.text=Number of occurrences
CommonAttributePanel.displayResultsLabel.text_2=Display results organized by:

29
Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstance.java
View File

@ -1,16 +1,16 @@
/*
*
*
* Autopsy Forensic Browser
*
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -31,30 +31,33 @@ import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Encapsulates data required to instantiate a <code>FileInstanceNode</code> for an instance in the CaseDB
* Encapsulates data required to instantiate a <code>FileInstanceNode</code> for
* an instance in the CaseDB
*/
final public class CaseDBCommonAttributeInstance extends AbstractCommonAttributeInstance {
private static final Logger LOGGER = Logger.getLogger(CaseDBCommonAttributeInstance.class.getName());
private final String value;
/**
* Create meta data required to find an abstract file and build a
* FileInstanceNode.
*
* @param objectId id of abstract file to find
* @param objectId id of abstract file to find
* @param dataSourceName name of datasource where the object is found
* @param value the correlation value which was found
*/
CaseDBCommonAttributeInstance(Long abstractFileReference, String dataSource, String caseName) {
CaseDBCommonAttributeInstance(Long abstractFileReference, String dataSource, String caseName, String value) {
super(abstractFileReference, dataSource, caseName);
this.value = value;
}
@Override
public DisplayableItemNode[] generateNodes() {
final CaseDBCommonAttributeInstanceNode intraCaseCommonAttributeInstanceNode = new CaseDBCommonAttributeInstanceNode(this.getAbstractFile(), this.getCaseName(), this.getDataSource());
final CaseDBCommonAttributeInstanceNode intraCaseCommonAttributeInstanceNode = new CaseDBCommonAttributeInstanceNode(this.getAbstractFile(), this.getCaseName(), this.getDataSource(), this.value, NODE_TYPE.COUNT_NODE);
return Arrays.asList(intraCaseCommonAttributeInstanceNode).toArray(new DisplayableItemNode[1]);
}
@Override
AbstractFile getAbstractFile() {

45
Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java
View File

@ -37,20 +37,26 @@ public class CaseDBCommonAttributeInstanceNode extends FileNode {
private final String caseName;
private final String dataSource;
private final String value;
private final AbstractCommonAttributeInstance.NODE_TYPE nodeType;
/**
* Create a node which can be used in a multilayer tree table and is based
* on an <code>AbstractFile</code>.
*
* @param fsContent the file which is being represented by this node
* @param caseName the name of the case
* @param fsContent the file which is being represented by this node
* @param caseName the name of the case
* @param dataSource the datasource which contains the file
*
* @param value the value that the correlation attribute was matched on
* @param nodeType the type of node to display columns for
*
*/
public CaseDBCommonAttributeInstanceNode(AbstractFile fsContent, String caseName, String dataSource) {
public CaseDBCommonAttributeInstanceNode(AbstractFile fsContent, String caseName, String dataSource, String value, AbstractCommonAttributeInstance.NODE_TYPE nodeType) {
super(fsContent, false);
this.caseName = caseName;
this.dataSource = dataSource;
this.nodeType = nodeType;
this.value = value;
}
@Override
@ -77,22 +83,27 @@ public class CaseDBCommonAttributeInstanceNode extends FileNode {
Sheet sheet = super.createSheet();
Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES);
Set<String> keepProps = new HashSet<>(Arrays.asList(
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType")));
for(Property<?> p : sheetSet.getProperties()) {
if(!keepProps.contains(p.getName())){
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"),
NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType")));
for (Property<?> p : sheetSet.getProperties()) {
if (!keepProps.contains(p.getName())) {
sheetSet.remove(p.getName());
}
}
final String NO_DESCR = Bundle.CommonFilesSearchResultsViewerTable_noDescText();
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, this.getDataSource()));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), NO_DESCR, caseName));
//add different columns for complete information depending on how nodes are structured in results
if (nodeType == AbstractCommonAttributeInstance.NODE_TYPE.COUNT_NODE) {
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), NO_DESCR, this.getContent().getParentPath()));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, this.getDataSource()));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl(), Bundle.CommonFilesSearchResultsViewerTable_caseColLbl(), NO_DESCR, caseName));
} else if (nodeType == AbstractCommonAttributeInstance.NODE_TYPE.CASE_NODE) {
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_localPath(), Bundle.CommonFilesSearchResultsViewerTable_localPath(), NO_DESCR, this.getContent().getParentPath()));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), NO_DESCR, this.value));
}
return sheet;
}
}
}

10
Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepoCommonAttributeInstance.java
View File

@ -43,13 +43,15 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
private static final Logger LOGGER = Logger.getLogger(CentralRepoCommonAttributeInstance.class.getName());
private final Integer crFileId;
private final NODE_TYPE nodeType;
private CorrelationAttributeInstance currentAttribute;
private final CorrelationAttributeInstance.Type correlationType;
CentralRepoCommonAttributeInstance(Integer attrInstId, CorrelationAttributeInstance.Type correlationType) {
CentralRepoCommonAttributeInstance(Integer attrInstId, CorrelationAttributeInstance.Type correlationType, NODE_TYPE nodeType) {
super();
this.crFileId = attrInstId;
this.correlationType = correlationType;
this.nodeType = nodeType;
}
@Override
@ -75,7 +77,6 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
// Only attempt to make the abstract file if the attribute is from the current case
if (currentCase.getName().equals(currentAttributeInstance.getCorrelationCase().getCaseUUID())) {
SleuthkitCase tskDb = currentCase.getSleuthkitCase();
// Find the correct data source
@ -96,8 +97,7 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
parentPath += File.separator;
}
parentPath = parentPath.replace("\\", "/");
final String whereClause = String.format("lower(name) = '%s' AND md5 = '%s' AND lower(parent_path) = '%s' AND data_source_obj_id = %s", fileName, currentAttribute.getCorrelationValue(), parentPath, dataSource.get().getId());
final String whereClause = String.format("lower(name) = '%s' AND lower(parent_path) = '%s' AND data_source_obj_id = %s", fileName, parentPath, dataSource.get().getId());
List<AbstractFile> potentialAbstractFiles = tskDb.findAllFilesWhere(whereClause);
if (potentialAbstractFiles.isEmpty()) {
@ -127,7 +127,7 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr
String currCaseDbName = Case.getCurrentCase().getDisplayName();
try {
AbstractFile abstractFileForAttributeInstance = this.getAbstractFile();
DisplayableItemNode generatedInstNode = AbstractCommonAttributeInstance.createNode(currentAttribute, abstractFileForAttributeInstance, currCaseDbName);
DisplayableItemNode generatedInstNode = AbstractCommonAttributeInstance.createNode(currentAttribute, abstractFileForAttributeInstance, currCaseDbName, nodeType);
attrInstNodeList.add(generatedInstNode);
} catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, String.format("Unable to get DataSource for record with md5: %s. Node not created.", new Object[]{currentAttribute.getCorrelationValue()}), ex);

79
Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepoCommonAttributeInstanceNode.java
View File

@ -1,16 +1,16 @@
/*
*
*
* Autopsy Forensic Browser
*
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -26,40 +26,45 @@ import java.util.List;
import javax.swing.Action;
import org.openide.nodes.Children;
import org.openide.nodes.Sheet;
import org.openide.util.NbBundle;
import org.openide.util.lookup.Lookups;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor;
import org.sleuthkit.autopsy.datamodel.NodeProperty;
/**
* Used by the Common Files search feature to encapsulate instances of a given
* MD5s matched in the search. These nodes will be children of <code>Md5Node</code>s.
*
* Use this type for files which are not in the current case, but from the
* Central Repo. Contrast with <code>SleuthkitCase</code> which should be used
* Used by the Common Files search feature to encapsulate instances of a given
* MD5s matched in the search. These nodes will be children of
* <code>Md5Node</code>s.
*
* Use this type for files which are not in the current case, but from the
* Central Repo. Contrast with <code>SleuthkitCase</code> which should be used
* when the FileInstance was found in the case presently open in Autopsy.
*/
public class CentralRepoCommonAttributeInstanceNode extends DisplayableItemNode {
private final CorrelationAttributeInstance crFile;
CentralRepoCommonAttributeInstanceNode(CorrelationAttributeInstance content) {
private final AbstractCommonAttributeInstance.NODE_TYPE nodeType;
CentralRepoCommonAttributeInstanceNode(CorrelationAttributeInstance content, AbstractCommonAttributeInstance.NODE_TYPE nodeType) {
super(Children.LEAF, Lookups.fixed(content));
this.crFile = content;
this.setDisplayName(new File(this.crFile.getFilePath()).getName());
this.nodeType = nodeType;
}
public CorrelationAttributeInstance getCorrelationAttributeInstance(){
public CorrelationAttributeInstance getCorrelationAttributeInstance() {
return this.crFile;
}
@Override
public Action[] getActions(boolean context){
public Action[] getActions(boolean context) {
List<Action> actionsList = new ArrayList<>();
actionsList.addAll(Arrays.asList(super.getActions(true)));
return actionsList.toArray(new Action[actionsList.size()]);
}
@ -79,19 +84,19 @@ public class CentralRepoCommonAttributeInstanceNode extends DisplayableItemNode
// of this type and they will need to provide the same key
return CaseDBCommonAttributeInstanceNode.class.getName();
}
@Override
protected Sheet createSheet(){
protected Sheet createSheet() {
Sheet sheet = new Sheet();
Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES);
if(sheetSet == null){
if (sheetSet == null) {
sheetSet = Sheet.createPropertiesSet();
sheet.put(sheetSet);
}
final CorrelationAttributeInstance centralRepoFile = this.getCorrelationAttributeInstance();
final String fullPath = centralRepoFile.getFilePath();
final File file = new File(fullPath);
@ -99,18 +104,24 @@ public class CentralRepoCommonAttributeInstanceNode extends DisplayableItemNode
final String name = file.getName();
final String parent = file.getParent();
final String value = centralRepoFile.getCorrelationValue();
final String dataSourceName = centralRepoFile.getCorrelationDataSource().getName();
final String NO_DESCR = Bundle.CommonFilesSearchResultsViewerTable_noDescText();
sheetSet.put(new NodeProperty<>(org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), NO_DESCR, name));
sheetSet.put(new NodeProperty<>(org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), NO_DESCR, parent));
sheetSet.put(new NodeProperty<>(org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, dataSourceName));
sheetSet.put(new NodeProperty<>(org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), org.sleuthkit.autopsy.commonfilesearch.Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), NO_DESCR, caseName));
return sheet;
final String NO_DESCR = Bundle.CommonFilesSearchResultsViewerTable_noDescText();
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NO_DESCR, name));
//add different columns for complete information depending on how nodes are structured in results
if (nodeType == AbstractCommonAttributeInstance.NODE_TYPE.COUNT_NODE) {
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), NO_DESCR, parent));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, dataSourceName));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl(), Bundle.CommonFilesSearchResultsViewerTable_caseColLbl(), NO_DESCR, caseName));
} else if (nodeType == AbstractCommonAttributeInstance.NODE_TYPE.CASE_NODE) {
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_localPath(), Bundle.CommonFilesSearchResultsViewerTable_localPath(), NO_DESCR, parent));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), NO_DESCR, value));
}
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NO_DESCR, ""));
return sheet;
}
}

266
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributeCaseSearchResults.java Normal file
View File

@ -0,0 +1,266 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilesearch;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Stores the results from the various types of common attribute searching
* Stores results based on how they are currently displayed in the UI
*/
final public class CommonAttributeCaseSearchResults {
private static final Logger LOGGER = Logger.getLogger(CommonAttributeCaseSearchResults.class.getName());
// maps instance count to list of attribute values.
private final Map<String, Map<String, CommonAttributeValueList>> caseNameToDataSources;
/**
* Create a values object which can be handed off to the node factories.
*
* @param metadata list of CommonAttributeValue indexed by case
* name
* @param percentageThreshold threshold to filter out files which are too
* common, value of 0 is disabled
* @param resultType The type of Correlation Attribute being
* searched for
* @param mimeTypesToFilterOn Set of mime types to include for intercase
* searches
*/
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, resultType.getId(), mimeTypesToFilterOn);
}
/**
* Create a values object which can be handed off to the node factories.
*
* @param metadata Map of Datasources and their
* commonAttributeValueLists indexed by case name
* @param percentageThreshold threshold to filter out files which are too
* common, value of 0 is disabled
*/
CommonAttributeCaseSearchResults(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold) {
this.caseNameToDataSources = filterMetadata(metadata, percentageThreshold, CorrelationAttributeInstance.FILES_TYPE_ID, new HashSet<>());
}
/**
* Find the child node whose children have the specified case name.
*
* This is a convenience method - you can also iterate over
* <code>getValues()</code>.
*
* @param caseName caseNameKey
*
* @return list of values which represent matches
*/
Map<String, CommonAttributeValueList> getAttributeValuesForCaseName(String caseName) {
return this.caseNameToDataSources.get(caseName);
}
/**
* Get an unmodifiable collection of values, indexed by case name, which
* represents the common attributes found in the search.
*
* @return map of cases to data sources and their list of matches
*/
public Map<String, Map<String, CommonAttributeValueList>> getMetadata() {
return Collections.unmodifiableMap(this.caseNameToDataSources);
}
/**
* Get an unmodifiable collection of values, indexed by case name, which
* represents the common attributes found in the search.
*
* Remove results which are not found in the portion of available data
* sources described by maximumPercentageThreshold.
*
* @param metadata the unfiltered metadata
* @param percentageThreshold the percentage threshold that a file should
* not be more common than
* @param resultTypeId the ID of the result type contained in the
* metadata
* @param mimeTypesToFilterOn the mimetypes to include in our results
*
* @return metadata
*/
private Map<String, Map<String, CommonAttributeValueList>> filterMetadata(Map<String, Map<String, CommonAttributeValueList>> metadata, int percentageThreshold, int resultTypeId, Set<String> mimeTypesToFilterOn) {
try {
final String currentCaseName;
try {
currentCaseName = Case.getCurrentCaseThrows().getDisplayName();
} catch (NoCurrentCaseException ex) {
throw new EamDbException("Unable to get current case while performing filtering", ex);
}
Map<String, CommonAttributeValueList> currentCaseDataSourceMap = metadata.get(currentCaseName);
if (currentCaseDataSourceMap == null) {
throw new EamDbException("No data for current case found in results, indicating there are no results and nothing will be filtered");
}
CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
.getDefaultCorrelationTypes()
.stream()
.filter(filterType -> filterType.getId() == resultTypeId)
.findFirst().get();
//Call countUniqueDataSources once to reduce the number of DB queries needed to get the frequencyPercentage
Double uniqueCaseDataSourceTuples = EamDb.getInstance().getCountUniqueDataSources().doubleValue();
Map<String, Map<String, CommonAttributeValueList>> filteredCaseNameToDataSourcesTree = new HashMap<>();
Map<String, CommonAttributeValue> valuesToKeepCurrentCase = getValuesToKeepFromCurrentCase(currentCaseDataSourceMap, attributeType, percentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn);
for (Entry<String, Map<String, CommonAttributeValueList>> mapOfDataSources : Collections.unmodifiableMap(metadata).entrySet()) {
if (!mapOfDataSources.getKey().equals(currentCaseName)) {
//rebuild the metadata structure with items from the current case substituted for their matches in other cases results we want to filter out removed
Map<String, CommonAttributeValueList> newTreeForCase = createTreeForCase(valuesToKeepCurrentCase, mapOfDataSources.getValue());
filteredCaseNameToDataSourcesTree.put(mapOfDataSources.getKey(), newTreeForCase);
}
}
return filteredCaseNameToDataSourcesTree;
} catch (EamDbException ex) {
LOGGER.log(Level.INFO, "Unable to perform filtering returning unfiltered result set", ex);
return metadata;
}
}
/**
* Get the values from the results for the current case
*
* @param dataSourceToValueList the map of datasources to their
* CommonAttributeValueLists for the
* current case
* @param attributeType the result type contained in the
* metadata
* @param maximumPercentageThreshold the percentage threshold that a file
* should not be more common than
* @param uniqueCaseDataSourceTuples the number of unique data sources in
* the CR
* @param mimeTypesToFilterOn the mimetypes to include in our results
*
* @return a map of correlation value to CommonAttributeValue for results
* from the current case
*
* @throws EamDbException
*/
private Map<String, CommonAttributeValue> getValuesToKeepFromCurrentCase(Map<String, CommonAttributeValueList> dataSourceToValueList, CorrelationAttributeInstance.Type attributeType, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToFilterOn) throws EamDbException {
Map<String, CommonAttributeValue> valuesToKeep = new HashMap<>();
Set<String> valuesToRemove = new HashSet<>();
for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
if (valuesToRemove.contains(value.getValue())) {
//do nothing this value will not be added
} else if (filterValue(attributeType, value, maximumPercentageThreshold, uniqueCaseDataSourceTuples, mimeTypesToFilterOn)) {
valuesToRemove.add(value.getValue());
} else {
valuesToKeep.put(value.getValue(), value);
}
}
}
return valuesToKeep;
}
/**
* Create a new map representing the portion of the tree for a single case
*
* @param valuesToKeepCurrentCase a map of correlation value to
* CommonAttributeValue for results from the
* current case to substitute in
* @param dataSourceToValueList the reslts for a single case which need to
* be filtered
*
* @return the modified results for the case
*
* @throws EamDbException
*/
private Map<String, CommonAttributeValueList> createTreeForCase(Map<String, CommonAttributeValue> valuesToKeepCurrentCase, Map<String, CommonAttributeValueList> dataSourceToValueList) throws EamDbException {
Map<String, CommonAttributeValueList> treeForCase = new HashMap<>();
for (Entry<String, CommonAttributeValueList> mapOfValueLists : Collections.unmodifiableMap(dataSourceToValueList).entrySet()) {
for (CommonAttributeValue value : mapOfValueLists.getValue().getDelayedMetadataList()) {
if (valuesToKeepCurrentCase.containsKey(value.getValue())) {
if (!treeForCase.containsKey(mapOfValueLists.getKey())) {
treeForCase.put(mapOfValueLists.getKey(), new CommonAttributeValueList());
}
treeForCase.get(mapOfValueLists.getKey()).addMetadataToList(valuesToKeepCurrentCase.get(value.getValue()));
}
}
}
return treeForCase;
}
/**
* Determine if a value should be included in the results displayed to the
* user
*
* @param attributeType the result type contained in the
* metadata
* @param value the correlationAttributeValue we are
* evaluating
* @param maximumPercentageThreshold the percentage threshold that a file
* should not be more common than
* @param uniqueCaseDataSourceTuples the number of unique data sources in
* the CR
* @param mimeTypesToInclude the mimetypes to include in our results
*
* @return true if the value should be filtered and removed from what is
* shown to the user, false if the value should not be removed and
* the user will see it as a result
*
* @throws EamDbException
*/
private boolean filterValue(CorrelationAttributeInstance.Type attributeType, CommonAttributeValue value, int maximumPercentageThreshold, Double uniqueCaseDataSourceTuples, Set<String> mimeTypesToInclude) throws EamDbException {
//Intracase common attribute searches will have been created with an empty mimeTypesToInclude list
//because when performing intra case search this filtering will have been done during the query of the case database
if (!mimeTypesToInclude.isEmpty()) { //only do the mime type filtering when mime types aren't empty
for (AbstractCommonAttributeInstance commonAttr : value.getInstances()) {
AbstractFile abstractFile = commonAttr.getAbstractFile();
if (abstractFile != null) {
String mimeType = abstractFile.getMIMEType();
if (mimeType != null && !mimeTypesToInclude.contains(mimeType)) {
return true;
}
}
}
}
if (maximumPercentageThreshold != 0) { //only do the frequency filtering when a max % was set
try {
Double uniqueTypeValueTuples = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(
attributeType, value.getValue()).doubleValue();
Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100;
int frequencyPercentage = commonalityPercentage.intValue();
if (frequencyPercentage > maximumPercentageThreshold) {
return true;
}
} catch (CorrelationAttributeNormalizationException ex) {
LOGGER.log(Level.WARNING, "Unable to determine frequency percentage attribute - frequency filter may not be accurate for these results.", ex);
}
}
return false;
}
}

32
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributeSearchResults.java → Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributeCountSearchResults.java
View File

@ -39,9 +39,9 @@ import org.sleuthkit.datamodel.AbstractFile;
* Stores the results from the various types of common attribute searching
* Stores results based on how they are currently displayed in the UI
*/
final public class CommonAttributeSearchResults {
final public class CommonAttributeCountSearchResults {
private static final Logger LOGGER = Logger.getLogger(CommonAttributeSearchResults.class.getName());
private static final Logger LOGGER = Logger.getLogger(CommonAttributeCountSearchResults.class.getName());
// maps instance count to list of attribute values.
private final Map<Integer, CommonAttributeValueList> instanceCountToAttributeValues;
@ -61,7 +61,7 @@ final public class CommonAttributeSearchResults {
* @param mimeTypesToFilterOn Set of mime types to include for intercase
* searches
*/
CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold, CorrelationAttributeInstance.Type resultType, Set<String> mimeTypesToFilterOn) {
//wrap in a new object in case any client code has used an unmodifiable collection
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
@ -77,7 +77,7 @@ final public class CommonAttributeSearchResults {
* @param percentageThreshold threshold to filter out files which are too
* common, value of 0 is disabled
*/
CommonAttributeSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold) {
CommonAttributeCountSearchResults(Map<Integer, CommonAttributeValueList> metadata, int percentageThreshold) {
//wrap in a new object in case any client code has used an unmodifiable collection
this.instanceCountToAttributeValues = new HashMap<>(metadata);
this.percentageThreshold = percentageThreshold;
@ -106,12 +106,17 @@ final public class CommonAttributeSearchResults {
*
* @return map of sizes of children to list of matches
*/
public Map<Integer, CommonAttributeValueList> getMetadata() throws EamDbException {
if (this.percentageThreshold == 0 && mimeTypesToInclude.isEmpty()) {
return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
} else {
return this.getMetadata(this.percentageThreshold);
}
public Map<Integer, CommonAttributeValueList> getMetadata() {
return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
}
/**
* Filter the results based on the criteria the user specified
*
* @throws EamDbException
*/
public void filterMetadata() throws EamDbException {
filterMetadata(this.percentageThreshold);
}
/**
@ -124,7 +129,7 @@ final public class CommonAttributeSearchResults {
*
* @return metadata
*/
private Map<Integer, CommonAttributeValueList> getMetadata(int maximumPercentageThreshold) throws EamDbException {
private void filterMetadata(int maximumPercentageThreshold) throws EamDbException {
CorrelationAttributeInstance.Type attributeType = CorrelationAttributeInstance
.getDefaultCorrelationTypes()
.stream()
@ -164,7 +169,7 @@ final public class CommonAttributeSearchResults {
//value will be removed as the mime type existed and was not in the set to be included
//because value is removed this value does not need to be checked further
mimeTypeToRemove = true;
break;
break;
}
}
if (mimeTypeToRemove) {
@ -193,7 +198,6 @@ final public class CommonAttributeSearchResults {
}
}
}
for (Entry<Integer, List<CommonAttributeValue>> valuesToRemove : itemsToRemove.entrySet()) {
final Integer key = valuesToRemove.getKey();
final List<CommonAttributeValue> values = valuesToRemove.getValue();
@ -207,8 +211,6 @@ final public class CommonAttributeSearchResults {
}
}
}
return Collections.unmodifiableMap(this.instanceCountToAttributeValues);
}
/**

66
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.form
View File

@ -4,6 +4,8 @@
<NonVisualComponents>
<Component class="javax.swing.ButtonGroup" name="interIntraButtonGroup">
</Component>
<Component class="javax.swing.ButtonGroup" name="displayResultsButtonGroup">
</Component>
</NonVisualComponents>
<Properties>
<Property name="minimumSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
@ -27,7 +29,7 @@
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
<AuxValue name="designerSize" type="java.awt.Dimension" value="-84,-19,0,5,115,114,0,18,106,97,118,97,46,97,119,116,46,68,105,109,101,110,115,105,111,110,65,-114,-39,-41,-84,95,68,20,2,0,2,73,0,6,104,101,105,103,104,116,73,0,5,119,105,100,116,104,120,112,0,0,2,108,0,0,1,-62"/>
<AuxValue name="designerSize" type="java.awt.Dimension" value="-84,-19,0,5,115,114,0,18,106,97,118,97,46,97,119,116,46,68,105,109,101,110,115,105,111,110,65,-114,-39,-41,-84,95,68,20,2,0,2,73,0,6,104,101,105,103,104,116,73,0,5,119,105,100,116,104,120,112,0,0,2,-122,0,0,1,-62"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.DesignBorderLayout"/>
@ -37,6 +39,9 @@
<Property name="maximumSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="null"/>
</Property>
<Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[450, 646]"/>
</Property>
<Property name="requestFocusEnabled" type="boolean" value="false"/>
</Properties>
<Constraints>
@ -94,6 +99,22 @@
</Group>
</Group>
</Group>
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="displayResultsLabel" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="30" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="caseResultsRadioButton" min="-2" pref="410" max="-2" attributes="0"/>
<Component id="countResultsRadioButton" max="32767" attributes="0"/>
</Group>
</Group>
</Group>
<EmptySpace min="-2" pref="10" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
@ -115,9 +136,15 @@
<Component id="percentageThresholdInputBox" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="percentageThresholdTextTwo" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="displayResultsLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="caseResultsRadioButton" min="-2" max="-2" attributes="0"/>
<EmptySpace min="0" pref="0" max="-2" attributes="0"/>
<Component id="countResultsRadioButton" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="dataSourcesLabel" min="-2" pref="14" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="searchButton" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="errorText" alignment="3" min="-2" max="-2" attributes="0"/>
@ -187,7 +214,7 @@
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<EmptySpace min="0" pref="330" max="32767" attributes="0"/>
<EmptySpace min="0" pref="326" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
</Layout>
@ -256,6 +283,37 @@
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="searchButtonActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JRadioButton" name="caseResultsRadioButton">
<Properties>
<Property name="buttonGroup" type="javax.swing.ButtonGroup" editor="org.netbeans.modules.form.RADComponent$ButtonGroupPropertyEditor">
<ComponentRef name="displayResultsButtonGroup"/>
</Property>
<Property name="selected" type="boolean" value="true"/>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/commonfilesearch/Bundle.properties" key="CommonAttributePanel.caseResultsRadioButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="caseResultsRadioButtonActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JRadioButton" name="countResultsRadioButton">
<Properties>
<Property name="buttonGroup" type="javax.swing.ButtonGroup" editor="org.netbeans.modules.form.RADComponent$ButtonGroupPropertyEditor">
<ComponentRef name="displayResultsButtonGroup"/>
</Property>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/commonfilesearch/Bundle.properties" key="CommonAttributePanel.countResultsRadioButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Component class="javax.swing.JLabel" name="displayResultsLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/commonfilesearch/Bundle.properties" key="CommonAttributePanel.displayResultsLabel.text_2" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Container>
</SubComponents>

194
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java
View File

@ -81,7 +81,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
private final UserInputErrorManager errorManager;
private int percentageThresholdValue = 20;
private final IntraCasePanel intraCasePanel;
private final InterCasePanel interCasePanel;
@ -97,17 +97,17 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
super(WindowManager.getDefault().getMainWindow(), Bundle.CommonAttributePanel_frame_title(), true);
initComponents();
this.setLocationRelativeTo(WindowManager.getDefault().getMainWindow());
interCasePanel = new InterCasePanel();
interCasePanel.setVisible(true);
interCasePanel.setSize((int) containerPanel.getPreferredSize().getWidth(), (int) containerPanel.getPreferredSize().getHeight());
intraCasePanel = new IntraCasePanel();
intraCasePanel.setVisible(true);
intraCasePanel.setSize((int) containerPanel.getPreferredSize().getWidth(), (int) containerPanel.getPreferredSize().getHeight());
this.setupDataSources();
if (CommonAttributePanel.isEamDbAvailableForIntercaseSearch()) {
this.setupCases();
this.interCasePanel.setupCorrelationTypeFilter();
@ -220,14 +220,14 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
"CommonAttributePanel.search.done.interupted=Something went wrong finding common properties.",
"CommonAttributePanel.search.done.sqlException=Unable to query db for properties or data sources.",
"CommonAttributePanel.search.done.noResults=No results found."})
private void search() {
new SwingWorker<CommonAttributeSearchResults, Void>() {
private void searchByCount() {
new SwingWorker<CommonAttributeCountSearchResults, Void>() {
private String tabTitle;
private ProgressHandle progress;
@Override
protected CommonAttributeSearchResults doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
protected CommonAttributeCountSearchResults doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
progress = ProgressHandle.createHandle(Bundle.CommonAttributePanel_search_done_searchProgressGathering());
progress.start();
progress.switchToIndeterminate();
@ -236,7 +236,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
Integer caseId = interCasePanel.getSelectedCaseId();
AbstractCommonAttributeSearcher builder;
CommonAttributeSearchResults metadata;
CommonAttributeCountSearchResults metadata;
boolean filterByMedia = false;
boolean filterByDocuments = false;
@ -276,7 +276,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
}
}
metadata = builder.findMatches();
metadata = builder.findMatchesByCount();
this.tabTitle = builder.getTabTitle();
return metadata;
}
@ -285,13 +285,8 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
protected void done() {
try {
super.done();
CommonAttributeSearchResults metadata = this.get();
boolean noKeysExist = true;
try {
noKeysExist = metadata.getMetadata().keySet().isEmpty();
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Unable to get keys from metadata", ex);
}
CommonAttributeCountSearchResults metadata = this.get();
boolean noKeysExist = metadata.getMetadata().keySet().isEmpty();
if (noKeysExist) {
Node commonFilesNode = new TableFilterNode(new EmptyNode(Bundle.CommonAttributePanel_search_done_noResults()), true);
progress.setDisplayName(Bundle.CommonAttributePanel_search_done_searchProgressDisplay());
@ -335,6 +330,108 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
}.execute();
}
/**
* Perform the common attribute search.
*/
private void searchByCase() {
new SwingWorker<CommonAttributeCaseSearchResults, Void>() {
private String tabTitle;
private ProgressHandle progress;
@Override
protected CommonAttributeCaseSearchResults doInBackground() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
progress = ProgressHandle.createHandle(Bundle.CommonAttributePanel_search_done_searchProgressGathering());
progress.start();
progress.switchToIndeterminate();
Long dataSourceId = intraCasePanel.getSelectedDataSourceId();
Integer caseId = interCasePanel.getSelectedCaseId();
AbstractCommonAttributeSearcher builder;
CommonAttributeCaseSearchResults metadata;
boolean filterByMedia = false;
boolean filterByDocuments = false;
int percentageThreshold = CommonAttributePanel.this.percentageThresholdValue;
if (!CommonAttributePanel.this.percentageThresholdCheck.isSelected()) {
//0 has the effect of disabling the feature
percentageThreshold = 0;
}
if (CommonAttributePanel.this.interCaseRadio.isSelected()) {
CorrelationAttributeInstance.Type corType = interCasePanel.getSelectedCorrelationType();
if (interCasePanel.fileCategoriesButtonIsSelected()) {
filterByMedia = interCasePanel.pictureVideoCheckboxIsSelected();
filterByDocuments = interCasePanel.documentsCheckboxIsSelected();
}
if (corType == null) {
corType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0);
}
if (caseId == InterCasePanel.NO_CASE_SELECTED) {
builder = new AllInterCaseCommonAttributeSearcher(filterByMedia, filterByDocuments, corType, percentageThreshold);
} else {
builder = new SingleInterCaseCommonAttributeSearcher(caseId, filterByMedia, filterByDocuments, corType, percentageThreshold);
}
} else {
if (intraCasePanel.fileCategoriesButtonIsSelected()) {
filterByMedia = intraCasePanel.pictureVideoCheckboxIsSelected();
filterByDocuments = intraCasePanel.documentsCheckboxIsSelected();
}
if (Objects.equals(dataSourceId, CommonAttributePanel.NO_DATA_SOURCE_SELECTED)) {
builder = new AllIntraCaseCommonAttributeSearcher(intraCasePanel.getDataSourceMap(), filterByMedia, filterByDocuments, percentageThreshold);
} else {
builder = new SingleIntraCaseCommonAttributeSearcher(dataSourceId, intraCasePanel.getDataSourceMap(), filterByMedia, filterByDocuments, percentageThreshold);
}
}
metadata = builder.findMatchesByCase();
this.tabTitle = builder.getTabTitle();
return metadata;
}
@Override
protected void done() {
try {
super.done();
CommonAttributeCaseSearchResults metadata = this.get();
if (metadata.getMetadata().keySet().isEmpty()) {
Node commonFilesNode = new TableFilterNode(new EmptyNode(Bundle.CommonAttributePanel_search_done_noResults()), true);
progress.setDisplayName(Bundle.CommonAttributePanel_search_done_searchProgressDisplay());
DataResultTopComponent.createInstance(tabTitle, Bundle.CommonAttributePanel_search_results_pathText(), commonFilesNode, 1);
} else {
// -3969
Node commonFilesNode = new CommonAttributeSearchResultRootNode(metadata);
DataResultFilterNode dataResultFilterNode = new DataResultFilterNode(commonFilesNode, ExplorerManager.find(CommonAttributePanel.this));
TableFilterNode tableFilterWithDescendantsNode = new TableFilterNode(dataResultFilterNode, 3);
DataResultViewerTable table = new CommonAttributesSearchResultsViewerTable();
Collection<DataResultViewer> viewers = new ArrayList<>(1);
viewers.add(table);
progress.setDisplayName(Bundle.CommonAttributePanel_search_done_searchProgressDisplay());
//0 passed as arguement due to JIRA-4502 ensuring the value is never displayed JIRA-TODO
DataResultTopComponent.createInstance(tabTitle, Bundle.CommonAttributePanel_search_results_pathText(), tableFilterWithDescendantsNode, 0, viewers);
}
} catch (InterruptedException ex) {
LOGGER.log(Level.SEVERE, "Interrupted while loading Common Files", ex);
MessageNotifyUtil.Message.error(Bundle.CommonAttributePanel_search_done_interupted());
} catch (ExecutionException ex) {
String errorMessage;
Throwable inner = ex.getCause();
if (inner instanceof TskCoreException) {
LOGGER.log(Level.SEVERE, "Failed to load files from database.", ex);
errorMessage = Bundle.CommonAttributePanel_search_done_tskCoreException();
} else if (inner instanceof NoCurrentCaseException) {
LOGGER.log(Level.SEVERE, "Current case has been closed.", ex);
errorMessage = Bundle.CommonAttributePanel_search_done_noCurrentCaseException();
} else if (inner instanceof SQLException) {
LOGGER.log(Level.SEVERE, "Unable to query db for files.", ex);
errorMessage = Bundle.CommonAttributePanel_search_done_sqlException();
} else {
LOGGER.log(Level.SEVERE, "Unexpected exception while running Common Files Search.", ex);
errorMessage = Bundle.CommonAttributePanel_search_done_exception();
}
MessageNotifyUtil.Message.error(errorMessage);
} finally {
progress.finish();
}
}
}.execute();
}
/**
* Sets up the data sources dropdown and returns the data sources map for
* future usage.
@ -424,15 +521,18 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
}
}.execute();
}
/**
* Display the provided panel inside the container panel.
*
*
* @param panel The panel to be shown.
*/
private void switchInnerPanel(JPanel panel) {
containerPanel.removeAll();
containerPanel.add(panel);
caseResultsRadioButton.setVisible(this.interCaseRadio.isSelected());
countResultsRadioButton.setVisible(this.interCaseRadio.isSelected());
displayResultsLabel.setVisible(this.interCaseRadio.isSelected());
this.revalidate();
this.repaint();
}
@ -519,6 +619,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
private void initComponents() {
interIntraButtonGroup = new javax.swing.ButtonGroup();
displayResultsButtonGroup = new javax.swing.ButtonGroup();
jPanel1 = new javax.swing.JPanel();
commonItemSearchDescription = new javax.swing.JLabel();
scopeLabel = new javax.swing.JLabel();
@ -531,6 +632,9 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
dataSourcesLabel = new javax.swing.JLabel();
errorText = new javax.swing.JLabel();
searchButton = new javax.swing.JButton();
caseResultsRadioButton = new javax.swing.JRadioButton();
countResultsRadioButton = new javax.swing.JRadioButton();
displayResultsLabel = new javax.swing.JLabel();
setMinimumSize(new java.awt.Dimension(450, 570));
setResizable(false);
@ -541,6 +645,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
});
jPanel1.setMaximumSize(null);
jPanel1.setPreferredSize(new java.awt.Dimension(450, 646));
jPanel1.setRequestFocusEnabled(false);
org.openide.awt.Mnemonics.setLocalizedText(commonItemSearchDescription, org.openide.util.NbBundle.getMessage(CommonAttributePanel.class, "CommonAttributePanel.commonItemSearchDescription.text")); // NOI18N
@ -577,7 +682,7 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
);
containerPanelLayout.setVerticalGroup(
containerPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGap(0, 330, Short.MAX_VALUE)
.addGap(0, 326, Short.MAX_VALUE)
);
org.openide.awt.Mnemonics.setLocalizedText(percentageThresholdCheck, org.openide.util.NbBundle.getMessage(CommonAttributePanel.class, "CommonAttributePanel.percentageThresholdCheck.text_1_1")); // NOI18N
@ -610,6 +715,20 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
}
});
displayResultsButtonGroup.add(caseResultsRadioButton);
caseResultsRadioButton.setSelected(true);
org.openide.awt.Mnemonics.setLocalizedText(caseResultsRadioButton, org.openide.util.NbBundle.getMessage(CommonAttributePanel.class, "CommonAttributePanel.caseResultsRadioButton.text")); // NOI18N
caseResultsRadioButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
caseResultsRadioButtonActionPerformed(evt);
}
});
displayResultsButtonGroup.add(countResultsRadioButton);
org.openide.awt.Mnemonics.setLocalizedText(countResultsRadioButton, org.openide.util.NbBundle.getMessage(CommonAttributePanel.class, "CommonAttributePanel.countResultsRadioButton.text")); // NOI18N
org.openide.awt.Mnemonics.setLocalizedText(displayResultsLabel, org.openide.util.NbBundle.getMessage(CommonAttributePanel.class, "CommonAttributePanel.displayResultsLabel.text_2")); // NOI18N
javax.swing.GroupLayout jPanel1Layout = new javax.swing.GroupLayout(jPanel1);
jPanel1.setLayout(jPanel1Layout);
jPanel1Layout.setHorizontalGroup(
@ -648,6 +767,17 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
.addGap(20, 20, 20)
.addComponent(interCaseRadio, javax.swing.GroupLayout.PREFERRED_SIZE, 383, javax.swing.GroupLayout.PREFERRED_SIZE))))
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
.addGroup(jPanel1Layout.createSequentialGroup()
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(jPanel1Layout.createSequentialGroup()
.addContainerGap()
.addComponent(displayResultsLabel))
.addGroup(jPanel1Layout.createSequentialGroup()
.addGap(30, 30, 30)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(caseResultsRadioButton, javax.swing.GroupLayout.PREFERRED_SIZE, 410, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(countResultsRadioButton, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
.addGap(10, 10, 10))
);
jPanel1Layout.setVerticalGroup(
jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@ -667,9 +797,15 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
.addComponent(percentageThresholdCheck)
.addComponent(percentageThresholdInputBox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(percentageThresholdTextTwo))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(displayResultsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(caseResultsRadioButton)
.addGap(0, 0, 0)
.addComponent(countResultsRadioButton)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(dataSourcesLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 14, javax.swing.GroupLayout.PREFERRED_SIZE)
.addGap(18, 18, 18)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(searchButton)
.addComponent(errorText, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
@ -706,6 +842,10 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
this.dispose();
}//GEN-LAST:event_searchButtonActionPerformed
private void caseResultsRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_caseResultsRadioButtonActionPerformed
// TODO add your handling code here:
}//GEN-LAST:event_caseResultsRadioButtonActionPerformed
/**
* If the settings reflect that a inter-case search is being performed,
* checks that the data sources in the current case have been processed with
@ -789,7 +929,11 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
performSearch = DialogDisplayer.getDefault().notify(descriptor) == NotifyDescriptor.YES_OPTION;
}
if (performSearch) {
search();
if (interCaseRadio.isSelected() && caseResultsRadioButton.isSelected()) {
searchByCase();
} else {
searchByCount();
}
}
} catch (InterruptedException | ExecutionException ex) {
LOGGER.log(Level.SEVERE, "Unexpected exception while looking for common properties", ex); //NON-NLS
@ -868,9 +1012,13 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer
}
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JRadioButton caseResultsRadioButton;
private javax.swing.JLabel commonItemSearchDescription;
private javax.swing.JPanel containerPanel;
private javax.swing.JRadioButton countResultsRadioButton;
private javax.swing.JLabel dataSourcesLabel;
private javax.swing.ButtonGroup displayResultsButtonGroup;
private javax.swing.JLabel displayResultsLabel;
private javax.swing.JLabel errorText;
private javax.swing.JRadioButton interCaseRadio;
private javax.swing.ButtonGroup interIntraButtonGroup;

81
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributeSearchResultRootNode.java
View File

@ -19,28 +19,30 @@
package org.sleuthkit.autopsy.commonfilesearch;
import java.util.List;
import java.util.logging.Level;
import java.util.Map;
import java.util.logging.Logger;
import org.openide.nodes.ChildFactory;
import org.openide.nodes.Children;
import org.openide.nodes.Node;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor;
/**
* Top-level node to store common file search results. Current structure is:
* - node for number of matches
* -- node for MD5/commmon attribute
* --- node for instance.
* Top-level node to store common file search results. Current structure is: -
* node for number of matches -- node for MD5/commmon attribute --- node for
* instance.
*/
final public class CommonAttributeSearchResultRootNode extends DisplayableItemNode {
CommonAttributeSearchResultRootNode(CommonAttributeSearchResults metadataList) {
CommonAttributeSearchResultRootNode(CommonAttributeCountSearchResults metadataList) {
super(Children.create(new InstanceCountNodeFactory(metadataList), true));
}
CommonAttributeSearchResultRootNode(CommonAttributeCaseSearchResults metadataList) {
super(Children.create(new InstanceCaseNodeFactory(metadataList), true));
}
@NbBundle.Messages({
"CommonFilesNode.getName.text=Common Files"})
@Override
@ -62,39 +64,70 @@ final public class CommonAttributeSearchResultRootNode extends DisplayableItemNo
public String getItemType() {
return getClass().getName();
}
/**
* Used to generate <code>InstanceCountNode</code>s.
*/
static class InstanceCountNodeFactory extends ChildFactory<Integer>{
static class InstanceCountNodeFactory extends ChildFactory<Integer> {
private static final Logger LOGGER = Logger.getLogger(InstanceCountNodeFactory.class.getName());
private final CommonAttributeSearchResults searchResults;
private final CommonAttributeCountSearchResults searchResults;
/**
* Build a factory which converts a <code>CommonAttributeSearchResults</code>
* object into <code>DisplayableItemNode</code>s.
* @param searchResults
* Build a factory which converts a
* <code>CommonAttributeCountSearchResults</code> object into
* <code>DisplayableItemNode</code>s.
*
* @param searchResults
*/
InstanceCountNodeFactory(CommonAttributeSearchResults searchResults){
InstanceCountNodeFactory(CommonAttributeCountSearchResults searchResults) {
this.searchResults = searchResults;
}
@Override
protected boolean createKeys(List<Integer> list) {
try {
list.addAll(this.searchResults.getMetadata().keySet());
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Unable to create keys.", ex);
}
list.addAll(this.searchResults.getMetadata().keySet());
return true;
}
@Override
protected Node createNodeForKey(Integer instanceCount){
CommonAttributeValueList attributeValues = this.searchResults.getAttributeValuesForInstanceCount(instanceCount);
protected Node createNodeForKey(Integer instanceCount) {
CommonAttributeValueList attributeValues = this.searchResults.getAttributeValuesForInstanceCount(instanceCount);
return new InstanceCountNode(instanceCount, attributeValues);
}
}
/**
* Used to generate <code>InstanceCaseNode</code>s.
*/
static class InstanceCaseNodeFactory extends ChildFactory<String> {
private static final Logger LOGGER = Logger.getLogger(InstanceCaseNodeFactory.class.getName());
private final CommonAttributeCaseSearchResults searchResults;
/**
* Build a factory which converts a
* <code>CommonAttributeCaseSearchResults</code> object into
* <code>DisplayableItemNode</code>s.
*
* @param searchResults
*/
InstanceCaseNodeFactory(CommonAttributeCaseSearchResults searchResults) {
this.searchResults = searchResults;
}
@Override
protected boolean createKeys(List<String> list) {
list.addAll(this.searchResults.getMetadata().keySet());
return true;
}
@Override
protected Node createNodeForKey(String caseName) {
Map<String, CommonAttributeValueList> dataSourceNameToInstances = this.searchResults.getAttributeValuesForCaseName(caseName);
return new InstanceCaseNode(caseName, dataSourceNameToInstances);
}
}
}

36
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributeValue.java
View File

@ -1,16 +1,16 @@
/*
*
*
* Autopsy Forensic Browser
*
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -28,33 +28,27 @@ import java.util.Set;
import java.util.stream.Collectors;
/**
* Defines a value that was in the common file search results
* as well as information about its instances.
* Defines a value that was in the common file search results as well as
* information about its instances.
*/
final public class CommonAttributeValue {
private final String md5;
private final String value;
private final List<AbstractCommonAttributeInstance> fileInstances;
CommonAttributeValue(String md5, List<AbstractCommonAttributeInstance> fileInstances) {
this.md5 = md5;
this.fileInstances = fileInstances;
}
CommonAttributeValue(String md5) {
this.md5 = md5;
CommonAttributeValue(String value) {
this.value = value;
this.fileInstances = new ArrayList<>();
}
public String getValue() {
return this.md5;
return this.value;
}
/**
* concatenate cases this value was seen into a single string
*
* @return
*
* @return
*/
public String getCases() {
return this.fileInstances.stream().map(AbstractCommonAttributeInstance::getCaseName).collect(Collectors.joining(", "));
@ -65,7 +59,7 @@ final public class CommonAttributeValue {
for (AbstractCommonAttributeInstance data : this.fileInstances) {
sources.add(data.getDataSource());
}
return String.join(", ", sources);
}
@ -78,7 +72,7 @@ final public class CommonAttributeValue {
}
/**
* How many distinct file instances exist for the MD5 represented by this
* How many distinct file instances exist for the value represented by this
* object?
*
* @return number of instances

44
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributeValueNode.java
View File

@ -1,16 +1,16 @@
/*
*
*
* Autopsy Forensic Browser
*
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -25,13 +25,14 @@ import org.openide.nodes.Children;
import org.openide.nodes.Node;
import org.openide.nodes.Sheet;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor;
import org.sleuthkit.autopsy.datamodel.NodeProperty;
/**
* Represents the layer in the tree for the value (such as MD5) that was in multiple places.
* Children are instances of that value.
* Represents the layer in the tree for the value (such as MD5) that was in
* multiple places. Children are instances of that value.
*/
public class CommonAttributeValueNode extends DisplayableItemNode {
@ -45,36 +46,37 @@ public class CommonAttributeValueNode extends DisplayableItemNode {
})
/**
* Create a Match node whose children will all have this object in common.
*
* @param data the common feature, and the children
*/
public CommonAttributeValueNode(CommonAttributeValue data) {
super(Children.create(
new FileInstanceNodeFactory(data), true));
this.commonFileCount = data.getInstanceCount();
this.cases = data.getCases();
// @@ We seem to be doing this string concat twice. We also do it in getDataSources()
this.dataSources = String.join(", ", data.getDataSources());
this.value = data.getValue();
this.setDisplayName(String.format(Bundle.CommonAttributeValueNode_CommonAttributeValueNode_format(), this.value));
this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/fileset-icon-16.png"); //NON-NLS
}
/**
* How many files are in common? This will be the number of children.
* How many files are in common? This will be the number of children.
*
* @return int
*/
int getCommonFileCount() {
return this.commonFileCount;
}
String getCases(){
String getCases() {
return this.cases;
}
/**
* Datasources where these matches occur.
*
* @return string delimited list of sources
*/
String getDataSources() {
@ -82,14 +84,17 @@ public class CommonAttributeValueNode extends DisplayableItemNode {
}
/**
* MD5 which is common to these matches
* @return string md5 hash
* Value which is common to these matches
*
* @return string the the value which is correlated on
*/
public String getValue() {
return this.value;
}
@NbBundle.Messages({"Md5Node.createSheet.noDescription= "})
@NbBundle.Messages({
"ValueNode.createSheet.noDescription= "
})
@Override
protected Sheet createSheet() {
Sheet sheet = new Sheet();
@ -99,16 +104,14 @@ public class CommonAttributeValueNode extends DisplayableItemNode {
sheet.put(sheetSet);
}
final String NO_DESCR = Bundle.Md5Node_createSheet_noDescription();
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), NO_DESCR, ""));
final String NO_DESCR = Bundle.ValueNode_createSheet_noDescription();
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, this.getDataSources()));
return sheet;
}
@Override
public <T> T accept(DisplayableItemNodeVisitor<T> visitor) {
return visitor.visit(this);
@ -141,12 +144,11 @@ public class CommonAttributeValueNode extends DisplayableItemNode {
list.addAll(this.descendants.getInstances());
return true;
}
@Override
protected Node[] createNodesForKey(AbstractCommonAttributeInstance searchResult) {
return searchResult.generateNodes();
}
}
}

24
Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributesSearchResultsViewerTable.java
View File

@ -29,6 +29,7 @@ import javax.swing.table.TableColumn;
import javax.swing.table.TableColumnModel;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable;
import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode;
/**
* <code>DataResultViewerTable</code> which overrides the default column header
@ -48,14 +49,17 @@ public class CommonAttributesSearchResultsViewerTable extends DataResultViewerTa
static {
Map<String, Integer> map = new HashMap<>();
map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260);
map.put(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), 260);
map.put(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"), 20);
map.put(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"), 20);
map.put(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"), 20);
map.put(Bundle.CommonFilesSearchResultsViewerTable_instancesColLbl(), 65);
map.put(Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), 300);
map.put(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), 200);
map.put(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl(), 200);
map.put(Bundle.CommonFilesSearchResultsViewerTable_localPath(), 200);
map.put(Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), 200);
map.put(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), 200);
map.put(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), 100);
map.put(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), 130);
map.put(Bundle.CommonFilesSearchResultsViewerTable_tagsColLbl1(), 300);
map.put(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), 130);
COLUMN_WIDTHS = Collections.unmodifiableMap(map);
}
@ -75,14 +79,12 @@ public class CommonAttributesSearchResultsViewerTable extends DataResultViewerTa
@NbBundle.Messages({
"CommonFilesSearchResultsViewerTable.noDescText= ",
"CommonFilesSearchResultsViewerTable.filesColLbl=Files",
"CommonFilesSearchResultsViewerTable.instancesColLbl=Instances",
"CommonFilesSearchResultsViewerTable.localPath=Parent Path in Current Case",
"CommonFilesSearchResultsViewerTable.pathColLbl=Parent Path",
"CommonFilesSearchResultsViewerTable.hashsetHitsColLbl=Hash Set Hits",
"CommonFilesSearchResultsViewerTable.caseColLbl1=Case",
"CommonFilesSearchResultsViewerTable.caseColLbl=Case",
"CommonFilesSearchResultsViewerTable.valueColLbl=Value",
"CommonFilesSearchResultsViewerTable.dataSourceColLbl=Data Source",
"CommonFilesSearchResultsViewerTable.mimeTypeColLbl=MIME Type",
"CommonFilesSearchResultsViewerTable.tagsColLbl1=Tags"
})
@Override
protected void setColumnWidths() {
@ -99,7 +101,7 @@ public class CommonAttributesSearchResultsViewerTable extends DataResultViewerTa
if (defaultWidth == null) {
column.setPreferredWidth(DEFAULT_WIDTH);
LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue));
LOGGER.log(Level.WARNING, String.format("Tried to set width on a column not supported by the CommonAttributesSearchResultsViewerTable: %s", headerValue));
} else {
column.setPreferredWidth(defaultWidth);
}

149
Core/src/org/sleuthkit/autopsy/commonfilesearch/InstanceCaseNode.java Normal file
View File

@ -0,0 +1,149 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilesearch;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.openide.nodes.ChildFactory;
import org.openide.nodes.Children;
import org.openide.nodes.Node;
import org.openide.nodes.Sheet;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor;
import org.sleuthkit.autopsy.datamodel.NodeProperty;
/**
* Node used to group results by case.
*/
public final class InstanceCaseNode extends DisplayableItemNode {
private static final Logger logger = Logger.getLogger(InstanceCaseNode.class.getName());
final private String caseName;
final private Map<String, CommonAttributeValueList> dataSourceToValueList;
/**
* Create a node with all instances for the given case, and the given
* selection of metadata.
*
* @param caseName the name of the case
* @param attributeValues the map of data sources to
* CommonAttributeValueLists to be included
*/
public InstanceCaseNode(String caseName, Map<String, CommonAttributeValueList> attributeValues) {
super(Children.create(new CommonAttributeDataSourceNodeFactory(attributeValues), true));
this.caseName = caseName;
this.dataSourceToValueList = attributeValues;
this.setDisplayName(this.caseName);
this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/briefcase.png"); //NON-NLS
}
/**
* Name of the case all child nodes are contained in.
*
* @return String case name
*/
String getCaseName() {
return this.caseName;
}
/**
* Creates the Children of this node. By doing this here instead of in the
* constructor, lazy creation of the Children is made possible.
*/
void createChildren() {
setChildren(Children.create(new CommonAttributeDataSourceNodeFactory(dataSourceToValueList), true));
}
/**
* Get a list of metadata for the datasources which are children of this
* object.
*
* @return List<Md5Metadata>
*/
Map<String, CommonAttributeValueList> getDataSourceToValueList() {
return this.dataSourceToValueList;
}
@Override
public <T> T accept(DisplayableItemNodeVisitor<T> visitor) {
return visitor.visit(this);
}
@Override
public boolean isLeafTypeNode() {
return false;
}
@Override
public String getItemType() {
return getClass().getName();
}
@Override
protected Sheet createSheet() {
Sheet sheet = new Sheet();
Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES);
if (sheetSet == null) {
sheetSet = Sheet.createPropertiesSet();
sheet.put(sheetSet);
}
final String NO_DESCR = Bundle.InstanceCountNode_createSheet_noDescription();
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_localPath(), Bundle.CommonFilesSearchResultsViewerTable_localPath(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), NO_DESCR, ""));
return sheet;
}
/**
* ChildFactory which builds InstanceDataSourceNode from the metadata data
* sources.
*/
static class CommonAttributeDataSourceNodeFactory extends ChildFactory<String> {
/**
* Map of data sources, each of which is a parent node matching a case
* name, containing children FileNodes.
*/
private final Map<String, CommonAttributeValueList> metadata;
CommonAttributeDataSourceNodeFactory(Map<String, CommonAttributeValueList> attributeValues) {
this.metadata = new HashMap<>();
this.metadata.putAll(attributeValues);
}
@Override
protected boolean createKeys(List<String> list) {
list.addAll(this.metadata.keySet());
return true;
}
@Override
protected Node createNodeForKey(String dataSourceName) {
return new InstanceDataSourceNode(dataSourceName, this.metadata.get(dataSourceName));
}
}
}

15
Core/src/org/sleuthkit/autopsy/commonfilesearch/InstanceCountNode.java
View File

@ -28,7 +28,9 @@ import org.openide.nodes.Children;
import org.openide.nodes.Node;
import org.openide.nodes.Sheet;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor;
import org.sleuthkit.autopsy.datamodel.NodeProperty;
@ -117,14 +119,17 @@ public final class InstanceCountNode extends DisplayableItemNode {
}
final String NO_DESCR = Bundle.InstanceCountNode_createSheet_noDescription();
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"), NO_DESCR, ""));
if (UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) {
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"), NO_DESCR, ""));
}
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_instancesColLbl(), Bundle.CommonFilesSearchResultsViewerTable_instancesColLbl(), NO_DESCR, this.getInstanceCount()));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl(), Bundle.CommonFilesSearchResultsViewerTable_caseColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_tagsColLbl1(), Bundle.CommonFilesSearchResultsViewerTable_tagsColLbl1(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NO_DESCR, ""));
return sheet;
}

4
Core/src/org/sleuthkit/autopsy/commonfilesearch/InstanceCountNodeTreeExpansionListener.java
View File

@ -47,6 +47,10 @@ final class InstanceCountNodeTreeExpansionListener implements TreeExpansionListe
if (instanceCountNode != null) {
instanceCountNode.createChildren();
}
final InstanceDataSourceNode instanceDataSourceNode = dataResultFilterNode.getLookup().lookup(InstanceDataSourceNode.class);
if (instanceDataSourceNode != null) {
instanceDataSourceNode.createChildren();
}
}
}
}

138
Core/src/org/sleuthkit/autopsy/commonfilesearch/InstanceDataSourceNode.java Normal file
View File

@ -0,0 +1,138 @@
/*
*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.commonfilesearch;
import java.util.List;
import org.openide.nodes.ChildFactory;
import org.openide.nodes.Children;
import org.openide.nodes.Node;
import org.openide.nodes.Sheet;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNode;
import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor;
import org.sleuthkit.autopsy.datamodel.NodeProperty;
/**
* Node used to group results by data source.
*/
public final class InstanceDataSourceNode extends DisplayableItemNode {
private static final Logger logger = Logger.getLogger(InstanceDataSourceNode.class.getName());
final private String dataSourceName;
final private CommonAttributeValueList dataSourceToValueList;
/**
* Create a node with all instances for the given data source, and the given
* selection of metadata.
*
* @param dataSourceName the name of the dataSource
* @param attributeValues the commonAttributeValueList containing the
* results
*/
public InstanceDataSourceNode(String dataSourceName, CommonAttributeValueList attributeValues) {
super(Children.create(new FileInstanceNodeFactory(attributeValues), true));
this.dataSourceName = dataSourceName;
this.dataSourceToValueList = attributeValues;
this.setDisplayName(this.dataSourceName);
this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/image.png"); //NON-NLS
}
/**
* Get the name of the data source
*
* @return String data source name
*/
String getDatasourceName() {
return this.dataSourceName;
}
/**
* Creates the Children of this node. By doing this here instead of in the
* constructor, lazy creation of the Children is made possible.
*/
void createChildren() {
dataSourceToValueList.displayDelayedMetadata();
setChildren(Children.create(new FileInstanceNodeFactory(dataSourceToValueList), true));
}
@Override
public boolean isLeafTypeNode() {
return false;
}
@Override
public String getItemType() {
return getClass().getName();
}
@Override
protected Sheet createSheet() {
Sheet sheet = new Sheet();
Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES);
if (sheetSet == null) {
sheetSet = Sheet.createPropertiesSet();
sheet.put(sheetSet);
}
final String NO_DESCR = Bundle.InstanceCountNode_createSheet_noDescription();
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_localPath(), Bundle.CommonFilesSearchResultsViewerTable_localPath(), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), NO_DESCR, ""));
sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), Bundle.CommonFilesSearchResultsViewerTable_valueColLbl(), NO_DESCR, ""));
return sheet;
}
@Override
public <T> T accept(DisplayableItemNodeVisitor<T> visitor) {
return visitor.visit(this);
}
/**
* ChildFactory which builds DisplayableItem from the metadata data
* sources.
*/
static class FileInstanceNodeFactory extends ChildFactory<AbstractCommonAttributeInstance> {
private final CommonAttributeValueList descendants;
FileInstanceNodeFactory(CommonAttributeValueList descendants) {
this.descendants = descendants;
}
@Override
protected boolean createKeys(List<AbstractCommonAttributeInstance> list) {
for (CommonAttributeValue value : descendants.getDelayedMetadataList()) {
list.addAll(value.getInstances());
}
return true;
}
@Override
protected Node[] createNodesForKey(AbstractCommonAttributeInstance searchResult) {
return searchResult.generateNodes();
}
}
}

2
Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseCommonAttributeSearcher.java
View File

@ -19,10 +19,10 @@
*/
package org.sleuthkit.autopsy.commonfilesearch;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance.Type;
/**
* Provides logic for selecting common files from all data sources and all cases

149
Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseSearchResultsProcessor.java
View File

@ -34,12 +34,13 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil;
import org.sleuthkit.autopsy.centralrepository.datamodel.InstanceTableCallback;
import org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeInstance.NODE_TYPE;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.HashUtility;
/**
* Used to process and return CorrelationCase md5s from the EamDB for
* Used to process and return CorrelationCase values from the EamDB for
* CommonFilesSearch.
*/
final class InterCaseSearchResultsProcessor {
@ -109,8 +110,8 @@ final class InterCaseSearchResultsProcessor {
try {
InterCaseCommonAttributeRowCallback instancetableCallback = new InterCaseCommonAttributeRowCallback();
EamDb DbManager = EamDb.getInstance();
DbManager.processInstanceTableWhere(correlationType, String.format("id = %s", attrbuteId), instancetableCallback);
EamDb dbManager = EamDb.getInstance();
dbManager.processInstanceTableWhere(correlationType, String.format("id = %s", attrbuteId), instancetableCallback);
return instancetableCallback.getCorrelationAttribute();
@ -121,20 +122,49 @@ final class InterCaseSearchResultsProcessor {
return null;
}
/**
* Given the current case, fins all intercase common files from the EamDb
* and builds maps of case name to maps of data source name to
* CommonAttributeValueList.
*
* @param currentCase The current TSK Case.
*
* @return map of Case name to Maps of Datasources and their
* CommonAttributeValueLists
*/
Map<String, Map<String, CommonAttributeValueList>> findInterCaseValuesByCase(Case currentCase) {
try {
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
}
/**
* Given the current case, fins all intercase common files from the EamDb
* and builds maps of obj id to md5 and case.
*
* @param currentCase The current TSK Case.
*/
Map<Integer, CommonAttributeValueList> findInterCaseCommonAttributeValues(Case currentCase) {
Map<Integer, CommonAttributeValueList> findInterCaseValuesByCount(Case currentCase) {
try {
InterCaseCommonAttributesCallback instancetableCallback = new InterCaseCommonAttributesCallback();
EamDb DbManager = EamDb.getInstance();
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = DbManager.getCase(currentCase).getID();
int caseId = dbManager.getCase(currentCase).getID();
DbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId,
dbManager.processInstanceTableWhere(correlationType, String.format(interCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue()),
instancetableCallback);
@ -154,13 +184,13 @@ final class InterCaseSearchResultsProcessor {
* @param currentCase The current TSK Case.
* @param singleCase The case of interest. Matches must exist in this case.
*/
Map<Integer, CommonAttributeValueList> findSingleInterCaseCommonAttributeValues(Case currentCase, CorrelationCase singleCase) {
Map<Integer, CommonAttributeValueList> findSingleInterCaseValuesByCount(Case currentCase, CorrelationCase singleCase) {
try {
InterCaseCommonAttributesCallback instancetableCallback = new InterCaseCommonAttributesCallback();
EamDb DbManager = EamDb.getInstance();
int caseId = DbManager.getCase(currentCase).getID();
InterCaseByCountCallback instancetableCallback = new InterCaseByCountCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
int targetCaseId = singleCase.getID();
DbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId,
dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback);
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
@ -170,10 +200,38 @@ final class InterCaseSearchResultsProcessor {
}
/**
* Callback to use with findInterCaseCommonAttributeValues which generates a
* list of md5s for common files search
* Given the current case, and a specific case of interest, finds common
* files which exist between cases from the EamDb. Builds map of case name
* to maps of data source name to CommonAttributeValueList.
*
* @param currentCase The current TSK Case.
*
* @return map of Case name to Maps of Datasources and their
* CommonAttributeValueLists
*
* @param currentCase The current TSK Case.
* @param singleCase The case of interest. Matches must exist in this case.
*/
private class InterCaseCommonAttributesCallback implements InstanceTableCallback {
Map<String, Map<String, CommonAttributeValueList>> findSingleInterCaseValuesByCase(Case currentCase, CorrelationCase singleCase) {
try {
InterCaseByCaseCallback instancetableCallback = new InterCaseByCaseCallback();
EamDb dbManager = EamDb.getInstance();
int caseId = dbManager.getCase(currentCase).getID();
int targetCaseId = singleCase.getID();
dbManager.processInstanceTableWhere(correlationType, String.format(singleInterCaseWhereClause, caseId,
TskData.FileKnown.KNOWN.getFileKnownValue(), caseId, targetCaseId), instancetableCallback);
return instancetableCallback.getInstanceCollatedCommonFiles();
} catch (EamDbException ex) {
LOGGER.log(Level.SEVERE, "Error accessing EamDb processing CaseInstancesTable.", ex);
}
return new HashMap<>();
}
/**
* Callback to use with findInterCaseValuesByCount which generates a list of
* md5s for common files search
*/
private class InterCaseByCountCallback implements InstanceTableCallback {
final Map<Integer, CommonAttributeValueList> instanceCollatedCommonFiles = new HashMap<>();
@ -242,7 +300,7 @@ final class InterCaseSearchResultsProcessor {
// we don't *have* all the information for the rows in the CR,
// so we need to consult the present case via the SleuthkitCase object
// Later, when the FileInstanceNode is built. Therefore, build node generators for now.
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType);
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.COUNT_NODE);
CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
searchResult.setCurrentAttributeInst(corrAttr);
commonAttributeValue.addInstance(searchResult);
@ -253,6 +311,55 @@ final class InterCaseSearchResultsProcessor {
}
}
/**
* Callback to use with findInterCaseValuesByCount which generates a list of
* md5s for common files search
*/
private class InterCaseByCaseCallback implements InstanceTableCallback {
final Map<String, Map<String, CommonAttributeValueList>> caseCollatedDataSourceCollections = new HashMap<>();
@Override
public void process(ResultSet resultSet) {
try {
while (resultSet.next()) {
int resultId = InstanceTableCallback.getId(resultSet);
String corValue = InstanceTableCallback.getValue(resultSet);
if (corValue == null || HashUtility.isNoDataMd5(corValue)) {
continue;
}
CorrelationCase correlationCase = EamDb.getInstance().getCaseById(InstanceTableCallback.getCaseId(resultSet));
String caseName = correlationCase.getDisplayName();
CorrelationDataSource correlationDatasource = EamDb.getInstance().getDataSourceById(correlationCase, InstanceTableCallback.getDataSourceId(resultSet));
// String dataSourceName = correlationDatasource.getName();
String dataSourceNameKey = correlationDatasource.getName() + correlationDatasource.getDataSourceObjectID();
if (!caseCollatedDataSourceCollections.containsKey(caseName)) {
caseCollatedDataSourceCollections.put(caseName, new HashMap<String, CommonAttributeValueList>());
}
Map<String, CommonAttributeValueList> dataSourceToFile = caseCollatedDataSourceCollections.get(caseName);
if (!dataSourceToFile.containsKey(dataSourceNameKey)) {
dataSourceToFile.put(dataSourceNameKey, new CommonAttributeValueList());
}
CommonAttributeValueList valueList = dataSourceToFile.get(dataSourceNameKey);
CentralRepoCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType, NODE_TYPE.CASE_NODE);
CorrelationAttributeInstance corrAttr = findSingleCorrelationAttribute(resultId);
searchResult.setCurrentAttributeInst(corrAttr);
CommonAttributeValue commonAttributeValue = new CommonAttributeValue(corValue);
commonAttributeValue.addInstance(searchResult);
valueList.addMetadataToList(commonAttributeValue);
dataSourceToFile.put(dataSourceNameKey, valueList);
caseCollatedDataSourceCollections.put(caseName, dataSourceToFile);
}
} catch (EamDbException | SQLException ex) {
LOGGER.log(Level.WARNING, "Error getting artifact instances from database.", ex); // NON-NLS
}
}
Map<String, Map<String, CommonAttributeValueList>> getInstanceCollatedCommonFiles() {
return Collections.unmodifiableMap(caseCollatedDataSourceCollections);
}
}
/**
* Callback to use with findSingleCorrelationAttribute which retrieves a
* single CorrelationAttribute from the EamDb.
@ -264,13 +371,13 @@ final class InterCaseSearchResultsProcessor {
@Override
public void process(ResultSet resultSet) {
try {
EamDb DbManager = EamDb.getInstance();
EamDb dbManager = EamDb.getInstance();
while (resultSet.next()) {
CorrelationCase correlationCase = DbManager.getCaseById(InstanceTableCallback.getCaseId(resultSet));
CorrelationDataSource dataSource = DbManager.getDataSourceById(correlationCase, InstanceTableCallback.getDataSourceId(resultSet));
CorrelationCase correlationCase = dbManager.getCaseById(InstanceTableCallback.getCaseId(resultSet));
CorrelationDataSource dataSource = dbManager.getDataSourceById(correlationCase, InstanceTableCallback.getDataSourceId(resultSet));
try {
correlationAttributeInstance = DbManager.getCorrelationAttributeInstance(correlationType,
correlationAttributeInstance = dbManager.getCorrelationAttributeInstance(correlationType,
correlationCase,
dataSource,
InstanceTableCallback.getValue(resultSet),

18
Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java
View File

@ -28,6 +28,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.SleuthkitCase.CaseDbQuery;
@ -36,8 +37,8 @@ import org.sleuthkit.datamodel.TskCoreException;
/**
*
* Generates a <code>List<CommonFilesMetadata></code> when
* <code>findMatches()</code> is called, which organizes files by md5 to prepare
* to display in viewer.
* <code>findMatchesByCount()</code> is called, which organizes files by md5 to
* prepare to display in viewer.
*
* This entire thing runs on a background thread where exceptions are handled.
*/
@ -99,7 +100,7 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt
* @throws SQLException
*/
@Override
public CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException {
public CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException {
Map<String, CommonAttributeValue> commonFiles = new HashMap<>();
final Case currentCase = Case.getCurrentCaseThrows();
@ -125,10 +126,10 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt
if (commonFiles.containsKey(md5)) {
final CommonAttributeValue commonAttributeValue = commonFiles.get(md5);
commonAttributeValue.addInstance(new CaseDBCommonAttributeInstance(objectId, dataSource, caseName));
commonAttributeValue.addInstance(new CaseDBCommonAttributeInstance(objectId, dataSource, caseName, md5));
} else {
final CommonAttributeValue commonAttributeValue = new CommonAttributeValue(md5);
commonAttributeValue.addInstance(new CaseDBCommonAttributeInstance(objectId, dataSource, caseName));
commonAttributeValue.addInstance(new CaseDBCommonAttributeInstance(objectId, dataSource, caseName, md5));
commonFiles.put(md5, commonAttributeValue);
}
}
@ -136,7 +137,12 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt
Map<Integer, CommonAttributeValueList> instanceCollatedCommonFiles = collateMatchesByNumberOfInstances(commonFiles);
return new CommonAttributeSearchResults(instanceCollatedCommonFiles, this.frequencyPercentageThreshold);
return new CommonAttributeCountSearchResults(instanceCollatedCommonFiles, this.frequencyPercentageThreshold);
}
@Override
public CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
throw new EamDbException("Not Supported at the moment");
}
/**

36
Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java
View File

@ -69,20 +69,38 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri
* @throws EamDbException
*/
@Override
public CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
public CommonAttributeCountSearchResults findMatchesByCount() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
CorrelationCase cCase = this.getCorrelationCaseFromId(this.corrleationCaseId);
this.correlationCaseName = cCase.getDisplayName();
return this.findFiles(cCase);
}
CommonAttributeSearchResults findFiles(CorrelationCase correlationCase) throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId);
this.correlationCaseName = correlationCase.getDisplayName();
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseCommonAttributeValues(Case.getCurrentCase(), correlationCase);
Map<Integer, CommonAttributeValueList> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCount(Case.getCurrentCase(), correlationCase);
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
return new CommonAttributeCountSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
}
/**
* Collect metadata required to render the tree table where matches must
* occur in the case with the given ID.
*
* @return business object needed to populate tree table with results
*
* @throws TskCoreException
* @throws NoCurrentCaseException
* @throws SQLException
* @throws EamDbException
*/
@Override
public CommonAttributeCaseSearchResults findMatchesByCase() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException {
CorrelationCase correlationCase = this.getCorrelationCaseFromId(this.corrleationCaseId);
this.correlationCaseName = correlationCase.getDisplayName();
InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType);
Map<String, Map<String, CommonAttributeValueList>> interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseValuesByCase(Case.getCurrentCase(), correlationCase);
Set<String> mimeTypesToFilterOn = getMimeTypesToFilterOn();
return new CommonAttributeCaseSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType, mimeTypesToFilterOn);
}
@NbBundle.Messages({
"# {0} - case name",

18
Core/src/org/sleuthkit/autopsy/core/UserPreferences.java
View File

@ -75,8 +75,6 @@ public final class UserPreferences {
public static final String SHOW_ONLY_CURRENT_USER_TAGS = "ShowOnlyCurrentUserTags";
public static final String HIDE_CENTRAL_REPO_COMMENTS_AND_OCCURRENCES = "HideCentralRepoCommentsAndOccurrences";
public static final String DISPLAY_TRANSLATED_NAMES = "DisplayTranslatedNames";
public static final String MAXIMUM_NUMBER_OF_RESULTS = "MaximumNumberOfResults";
private static final int DEFAULT_MAX_RESULTS = 20000;
// Prevent instantiation.
private UserPreferences() {
@ -473,20 +471,4 @@ public final class UserPreferences {
public static void setLogFileCount(int count) {
preferences.putInt(MAX_NUM_OF_LOG_FILE, count);
}
/**
* Set the maximum number of result rows to show in data result tables.
* @param max
*/
public static void setMaximumNumberOfResults(int max) {
preferences.putInt(MAXIMUM_NUMBER_OF_RESULTS, max);
}
/**
* Get the maximum number of result rows to show in data result tables.
* @return
*/
public static int getMaximumNumberOfResults() {
return preferences.getInt(MAXIMUM_NUMBER_OF_RESULTS, DEFAULT_MAX_RESULTS);
}
}

2
Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties
View File

@ -176,8 +176,6 @@ ViewPreferencesPanel.globalSettingsPanel.border.title=Global Settings
ViewPreferencesPanel.translateNamesInTableRadioButton.text=Table
ViewPreferencesPanel.commentsOccurencesColumnWrapAroundText.text=to reduce loading times
ViewPreferencesPanel.translateTextLabel.text=Translate text in the:
ViewPreferencesPanel.maximumResultsLabel.toolTipText=Maximum numer of rows to display in result tables. 0 = unlimited
ViewPreferencesPanel.maximumResultsLabel.text=Maximum number of results to display:
ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccurences) columns
ViewPreferencesPanel.centralRepoLabel.text=Do not use Central Repository for:
ViewPreferencesPanel.hideOtherUsersTagsLabel.text=Hide other users' tags in the:

54
Core/src/org/sleuthkit/autopsy/corecomponents/TableFilterChildren.java
View File

@ -18,27 +18,18 @@
*/
package org.sleuthkit.autopsy.corecomponents;
import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;
import org.openide.nodes.Children;
import org.openide.nodes.FilterNode;
import org.openide.nodes.Node;
import org.openide.util.NbBundle;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.core.UserPreferences;
/**
* A <code>Children</code> implementation for a
* <code>TableFilterNode</code>. A
* <code>TableFilterNode</code> creates at most one layer of child
* nodes for the node it wraps. It is designed to be used in the results view
* to ensure the individual viewers display only the first layer of child nodes.
* A <code>Children</code> implementation for a <code>TableFilterNode</code>. A
* <code>TableFilterNode</code> creates at most one layer of child nodes for the
* node it wraps. It is designed to be used in the results view to ensure the
* individual viewers display only the first layer of child nodes.
*/
class TableFilterChildren extends FilterNode.Children {
private int numberOfNodesCreated;
private static volatile boolean maxResultsDialogShown = false;
/**
* Creates a Children object for a TableFilterNode. A TableFilterNode
* creates at most one layer of child nodes for the node it wraps. It is
@ -71,7 +62,6 @@ class TableFilterChildren extends FilterNode.Children {
*/
TableFilterChildren(Node wrappedNode) {
super(wrappedNode);
numberOfNodesCreated = 0;
}
/**
@ -96,41 +86,7 @@ class TableFilterChildren extends FilterNode.Children {
* @return
*/
@Override
@NbBundle.Messages({"# {0} - The results limit",
"TableFilterChildren.createNodes.limitReached.msg="
+ "The limit on the number of results to display has been reached."
+ " Only the first {0} results will be shown."
+ " The limit can be modified under Tools, Options, View."})
protected Node[] createNodes(Node key) {
int maxNodesToCreate = UserPreferences.getMaximumNumberOfResults();
if (maxNodesToCreate == 0 || numberOfNodesCreated < maxNodesToCreate) {
// We either haven't hit the limit yet, or we don't have a limit.
/**
* We only want to apply the limit to "our" nodes (i.e. not the
* wait node). If we don't do this the "Please wait..."
* node causes the number of results in the table to be off by one.
* Using the Bundle to get the value so that we are not tied to a
* particular locale.
*/
if (!key.getDisplayName().equalsIgnoreCase(NbBundle.getMessage(Node.class, "LBL_WAIT"))) {
numberOfNodesCreated++;
// If we have a limit and the creation of this node reaches it,
// tell the user if they haven't already been told.
if (numberOfNodesCreated == maxNodesToCreate && !maxResultsDialogShown) {
maxResultsDialogShown = true;
SwingUtilities.invokeLater(()
-> JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(),
Bundle.TableFilterChildren_createNodes_limitReached_msg(maxNodesToCreate))
);
}
}
return new Node[]{this.copyNode(key)};
} else {
return new Node[]{};
}
return new Node[]{this.copyNode(key)};
}
}

32
Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form
View File

@ -26,7 +26,7 @@
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="viewPreferencesScrollPane" alignment="0" pref="528" max="32767" attributes="0"/>
<Component id="viewPreferencesScrollPane" alignment="0" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
</Layout>
@ -153,11 +153,6 @@
</Group>
</Group>
</Group>
<Group type="102" alignment="0" attributes="0">
<Component id="maximumResultsLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="maximumResultsSpinner" min="-2" pref="70" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace max="32767" attributes="0"/>
</Group>
@ -190,11 +185,6 @@
<Component id="commentsOccurencesColumnsCheckbox" min="-2" pref="18" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="commentsOccurencesColumnWrapAroundText" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="6" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="maximumResultsLabel" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="maximumResultsSpinner" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<Group type="102" alignment="0" attributes="0">
<Component id="selectFileLabel" min="-2" max="-2" attributes="0"/>
@ -371,16 +361,6 @@
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="commentsOccurencesColumnsCheckboxActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JLabel" name="maximumResultsLabel">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/corecomponents/Bundle.properties" key="ViewPreferencesPanel.maximumResultsLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/corecomponents/Bundle.properties" key="ViewPreferencesPanel.maximumResultsLabel.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
</Component>
<Container class="javax.swing.JScrollPane" name="jScrollPane1">
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
@ -427,16 +407,6 @@
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="translateNamesInTableRadioButtonActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JSpinner" name="maximumResultsSpinner">
<Properties>
<Property name="model" type="javax.swing.SpinnerModel" editor="org.netbeans.modules.form.editors2.SpinnerModelEditor">
<SpinnerModel initial="20000" maximum="100000" minimum="0" numberType="java.lang.Integer" stepSize="10000" type="number"/>
</Property>
</Properties>
<Events>
<EventHandler event="stateChanged" listener="javax.swing.event.ChangeListener" parameters="javax.swing.event.ChangeEvent" handler="maximumResultsSpinnerStateChanged"/>
</Events>
</Component>
</SubComponents>
</Container>
<Container class="javax.swing.JPanel" name="currentCaseSettingsPanel">

38
Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java
View File

@ -78,7 +78,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
commentsOccurencesColumnsCheckbox.setEnabled(EamDbUtil.useCentralRepo());
commentsOccurencesColumnWrapAroundText.setEnabled(EamDbUtil.useCentralRepo());
commentsOccurencesColumnsCheckbox.setSelected(UserPreferences.hideCentralRepoCommentsAndOccurrences());
maximumResultsSpinner.setValue(UserPreferences.getMaximumNumberOfResults());
hideOtherUsersTagsCheckbox.setSelected(UserPreferences.showOnlyCurrentUserTags());
translateNamesInTableRadioButton.setSelected(UserPreferences.displayTranslatedFileNames());
@ -115,7 +114,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
UserPreferences.setShowOnlyCurrentUserTags(hideOtherUsersTagsCheckbox.isSelected());
UserPreferences.setHideCentralRepoCommentsAndOccurrences(commentsOccurencesColumnsCheckbox.isSelected());
UserPreferences.setDisplayTranslatedFileNames(translateNamesInTableRadioButton.isSelected());
UserPreferences.setMaximumNumberOfResults((Integer)maximumResultsSpinner.getValue());
storeGroupItemsInTreeByDataSource();
@ -164,13 +162,11 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
hideOtherUsersTagsLabel = new javax.swing.JLabel();
centralRepoLabel = new javax.swing.JLabel();
commentsOccurencesColumnsCheckbox = new javax.swing.JCheckBox();
maximumResultsLabel = new javax.swing.JLabel();
jScrollPane1 = new javax.swing.JScrollPane();
timeZoneList = new javax.swing.JList<>();
translateTextLabel = new javax.swing.JLabel();
commentsOccurencesColumnWrapAroundText = new javax.swing.JLabel();
translateNamesInTableRadioButton = new javax.swing.JRadioButton();
maximumResultsSpinner = new javax.swing.JSpinner();
currentCaseSettingsPanel = new javax.swing.JPanel();
groupByDataSourceCheckbox = new javax.swing.JCheckBox();
currentSessionSettingsPanel = new javax.swing.JPanel();
@ -270,9 +266,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
}
});
org.openide.awt.Mnemonics.setLocalizedText(maximumResultsLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.maximumResultsLabel.text")); // NOI18N
maximumResultsLabel.setToolTipText(org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.maximumResultsLabel.toolTipText")); // NOI18N
timeZoneList.addListSelectionListener(new javax.swing.event.ListSelectionListener() {
public void valueChanged(javax.swing.event.ListSelectionEvent evt) {
timeZoneListValueChanged(evt);
@ -291,13 +284,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
}
});
maximumResultsSpinner.setModel(new javax.swing.SpinnerNumberModel(20000, 0, 100000, 10000));
maximumResultsSpinner.addChangeListener(new javax.swing.event.ChangeListener() {
public void stateChanged(javax.swing.event.ChangeEvent evt) {
maximumResultsSpinnerStateChanged(evt);
}
});
javax.swing.GroupLayout globalSettingsPanelLayout = new javax.swing.GroupLayout(globalSettingsPanel);
globalSettingsPanel.setLayout(globalSettingsPanelLayout);
globalSettingsPanelLayout.setHorizontalGroup(
@ -347,11 +333,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
.addComponent(useBestViewerRadioButton)
.addComponent(useLocalTimeRadioButton)
.addComponent(useAnotherTimeRadioButton)
.addComponent(translateNamesInTableRadioButton)))))
.addGroup(globalSettingsPanelLayout.createSequentialGroup()
.addComponent(maximumResultsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(maximumResultsSpinner, javax.swing.GroupLayout.PREFERRED_SIZE, 70, javax.swing.GroupLayout.PREFERRED_SIZE)))
.addComponent(translateNamesInTableRadioButton))))))
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
);
globalSettingsPanelLayout.setVerticalGroup(
@ -380,11 +362,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
.addGap(3, 3, 3)
.addComponent(commentsOccurencesColumnsCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 18, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(commentsOccurencesColumnWrapAroundText)
.addGap(6, 6, 6)
.addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(maximumResultsLabel)
.addComponent(maximumResultsSpinner, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)))
.addComponent(commentsOccurencesColumnWrapAroundText))
.addGroup(globalSettingsPanelLayout.createSequentialGroup()
.addComponent(selectFileLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
@ -489,7 +467,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(viewPreferencesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 528, Short.MAX_VALUE)
.addComponent(viewPreferencesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
);
}// </editor-fold>//GEN-END:initComponents
@ -615,14 +593,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
}
}//GEN-LAST:event_useBestViewerRadioButtonActionPerformed
private void maximumResultsSpinnerStateChanged(javax.swing.event.ChangeEvent evt) {//GEN-FIRST:event_maximumResultsSpinnerStateChanged
if (immediateUpdates) {
UserPreferences.setMaximumNumberOfResults((Integer)maximumResultsSpinner.getValue());
} else {
firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
}
}//GEN-LAST:event_maximumResultsSpinnerStateChanged
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JLabel centralRepoLabel;
@ -642,8 +612,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel {
private javax.swing.JLabel hideSlackFilesLabel;
private javax.swing.JScrollPane jScrollPane1;
private javax.swing.JRadioButton keepCurrentViewerRadioButton;
private javax.swing.JLabel maximumResultsLabel;
private javax.swing.JSpinner maximumResultsSpinner;
private javax.swing.JLabel selectFileLabel;
private javax.swing.JList<String> timeZoneList;
private javax.swing.JRadioButton translateNamesInTableRadioButton;

4
Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java
View File

@ -469,7 +469,7 @@ public abstract class AbstractAbstractFileNode<T extends AbstractFile> extends A
"AbstractAbstractFileNode.createSheet.noScore.description=No score"})
Pair<DataResultViewerTable.Score, String> getScorePropertyAndDescription(List<ContentTag> tags) {
DataResultViewerTable.Score score = DataResultViewerTable.Score.NO_SCORE;
String description = "";
String description = Bundle.AbstractAbstractFileNode_createSheet_noScore_description();
if (content.getKnown() == TskData.FileKnown.BAD) {
score = DataResultViewerTable.Score.NOTABLE_SCORE;
description = Bundle.AbstractAbstractFileNode_createSheet_notableFile_description();
@ -572,7 +572,7 @@ public abstract class AbstractAbstractFileNode<T extends AbstractFile> extends A
CorrelationAttributeInstance getCorrelationAttributeInstance() {
CorrelationAttributeInstance attribute = null;
if (EamDbUtil.useCentralRepo()) {
if (EamDbUtil.useCentralRepo() && !UserPreferences.hideCentralRepoCommentsAndOccurrences()) {
attribute = EamArtifactUtil.getInstanceFromContent(content);
}
return attribute;

4
Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java
View File

@ -355,12 +355,12 @@ public class BlackboardArtifactNode extends AbstractContentNode<BlackboardArtifa
addScoreProperty(sheetSet, tags);
CorrelationAttributeInstance correlationAttribute = null;
if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) {
if (UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) {
correlationAttribute = getCorrelationAttributeInstance();
}
addCommentProperty(sheetSet, tags, correlationAttribute);
if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) {
if (UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) {
addCountProperty(sheetSet, correlationAttribute);
}
if (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT.getTypeID()) {

3
Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java
View File

@ -446,9 +446,6 @@ public class DeletedContent implements AutopsyVisitableItem {
if (Objects.equals(CasePreferences.getGroupItemsInTreeByDataSource(), true)) {
query += " AND data_source_obj_id = " + filteringDSObjId;
}
if (UserPreferences.getMaximumNumberOfResults() != 0) {
query += " LIMIT " + UserPreferences.getMaximumNumberOfResults(); //NON-NLS
}
return query;
}

38
Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011 - 2017 Basis Technology Corp.
* Copyright 2011 - 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -21,8 +21,10 @@ package org.sleuthkit.autopsy.datamodel;
import org.sleuthkit.autopsy.commonfilesearch.CentralRepoCommonAttributeInstanceNode;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResultRootNode;
import org.sleuthkit.autopsy.commonfilesearch.InstanceCountNode;
import org.sleuthkit.autopsy.commonfilesearch.InstanceCaseNode;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeValueNode;
import org.sleuthkit.autopsy.commonfilesearch.CaseDBCommonAttributeInstanceNode;
import org.sleuthkit.autopsy.commonfilesearch.InstanceDataSourceNode;
import org.sleuthkit.autopsy.datamodel.DeletedContent.DeletedContentsChildren.DeletedContentNode;
import org.sleuthkit.autopsy.datamodel.DeletedContent.DeletedContentsNode;
import org.sleuthkit.autopsy.datamodel.FileSize.FileSizeRootChildren.FileSizeNode;
@ -67,7 +69,7 @@ public interface DisplayableItemNodeVisitor<T> {
T visit(ViewsNode vn);
T visit(DataSourceGroupingNode dataSourceGroupingNode);
T visit(org.sleuthkit.autopsy.datamodel.FileTypesByExtension.FileExtensionNode fsfn);
T visit(DeletedContentNode dcn);
@ -122,11 +124,15 @@ public interface DisplayableItemNodeVisitor<T> {
T visit(CommonAttributeSearchResultRootNode cfn);
T visit(CaseDBCommonAttributeInstanceNode fin);
T visit(CentralRepoCommonAttributeInstanceNode crfin);
T visit(InstanceCountNode icn);
T visit(InstanceCaseNode icn);
T visit(InstanceDataSourceNode icn);
T visit(CorrelationAttributeInstanceNode cain);
/*
@ -211,19 +217,29 @@ public interface DisplayableItemNodeVisitor<T> {
public T visit(CommonAttributeSearchResultRootNode cfn) {
return defaultVisit(cfn);
}
@Override
public T visit(InstanceCountNode icn){
public T visit(InstanceCountNode icn) {
return defaultVisit(icn);
}
@Override
public T visit(InstanceCaseNode icn) {
return defaultVisit(icn);
}
@Override
public T visit(InstanceDataSourceNode icn) {
return defaultVisit(icn);
}
@Override
public T visit(CorrelationAttributeInstanceNode cain) {
return defaultVisit(cain);
}
@Override
public T visit(CentralRepoCommonAttributeInstanceNode crfin){
public T visit(CentralRepoCommonAttributeInstanceNode crfin) {
return defaultVisit(crfin);
}
@ -361,7 +377,7 @@ public interface DisplayableItemNodeVisitor<T> {
public T visit(DataSourceGroupingNode dataSourceGroupingNode) {
return defaultVisit(dataSourceGroupingNode);
}
@Override
public T visit(ResultsNode rn) {
return defaultVisit(rn);

36
Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java
View File

@ -55,6 +55,8 @@ import org.sleuthkit.autopsy.datamodel.FileNode;
import org.sleuthkit.autopsy.datamodel.FileTypeExtensions;
import org.sleuthkit.autopsy.datamodel.FileTypes.FileTypesNode;
import org.sleuthkit.autopsy.commonfilesearch.InstanceCountNode;
import org.sleuthkit.autopsy.commonfilesearch.InstanceCaseNode;
import org.sleuthkit.autopsy.commonfilesearch.InstanceDataSourceNode;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeValueNode;
import org.sleuthkit.autopsy.commonfilesearch.CentralRepoCommonAttributeInstanceNode;
import org.sleuthkit.autopsy.datamodel.LayoutFileNode;
@ -132,8 +134,8 @@ public class DataResultFilterNode extends FilterNode {
* wrapped node and may filter out some of its children.
*
* @param node The node to wrap.
* @param em The ExplorerManager for the component that is creating the
* node.
* @param em The ExplorerManager for the component that is creating the
* node.
*/
public DataResultFilterNode(Node node, ExplorerManager em) {
super(node, new DataResultFilterChildren(node, em));
@ -145,13 +147,13 @@ public class DataResultFilterNode extends FilterNode {
* result viewers. The wrapper node defines the actions associated with the
* wrapped node and may filter out some of its children.
*
* @param node The node to wrap.
* @param em The ExplorerManager for the component that is creating the
* node.
* @param node The node to wrap.
* @param em The ExplorerManager for the component that is creating
* the node.
* @param filterKnown Whether or not to filter out children that represent
* known files.
* known files.
* @param filterSlack Whether or not to filter out children that represent
* virtual slack space files.
* virtual slack space files.
*/
private DataResultFilterNode(Node node, ExplorerManager em, boolean filterKnown, boolean filterSlack) {
super(node, new DataResultFilterChildren(node, em, filterKnown, filterSlack));
@ -261,7 +263,7 @@ public class DataResultFilterNode extends FilterNode {
* selected.
*
* @return The child node selection information, or null if no child should
* be selected.
* be selected.
*/
public NodeSelectionInfo getChildNodeSelectionInfo() {
if (getOriginal() instanceof DisplayableItemNode) {
@ -530,20 +532,30 @@ public class DataResultFilterNode extends FilterNode {
}
@Override
public AbstractAction visit(CommonAttributeValueNode md5n){
public AbstractAction visit(InstanceCaseNode icn) {
return null;
}
@Override
public AbstractAction visit(CaseDBCommonAttributeInstanceNode fin){
public AbstractAction visit(InstanceDataSourceNode icn) {
return null;
}
@Override
public AbstractAction visit(CentralRepoCommonAttributeInstanceNode iccan){
public AbstractAction visit(CommonAttributeValueNode md5n) {
return null;
}
@Override
public AbstractAction visit(CaseDBCommonAttributeInstanceNode fin) {
return null;
}
@Override
public AbstractAction visit(CentralRepoCommonAttributeInstanceNode iccan) {
return null;
}
@Override
public AbstractAction visit(BlackboardArtifactNode ban) {
BlackboardArtifact artifact = ban.getArtifact();

1
Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java
View File

@ -173,7 +173,6 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
case UserPreferences.HIDE_CENTRAL_REPO_COMMENTS_AND_OCCURRENCES:
case UserPreferences.DISPLAY_TRANSLATED_NAMES:
case UserPreferences.KEEP_PREFERRED_VIEWER:
case UserPreferences.MAXIMUM_NUMBER_OF_RESULTS:
refreshContentTreeSafe();
break;
case UserPreferences.SHOW_ONLY_CURRENT_USER_TAGS:

BIN
Core/src/org/sleuthkit/autopsy/images/briefcase.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 288 B

11
Core/src/org/sleuthkit/autopsy/modules/dataSourceIntegrity/DataSourceIntegrityIngestModule.java
View File

@ -105,7 +105,11 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
"# {0} - hashAlgorithm",
"# {1} - calculatedHashValue",
"# {2} - storedHashValue",
"DataSourceIntegrityIngestModule.process.hashFailedForArtifact={0} hash verification failed:\n Calculated hash: {1}\n Stored hash: {2}\n",
"DataSourceIntegrityIngestModule.process.hashFailedForArtifact={0} hash verification failed:\n Calculated hash: {1}\n Stored hash: {2}\n",
"# {0} - imageName",
"DataSourceIntegrityIngestModule.process.verificationSuccess=Integrity of {0} verified",
"# {0} - imageName",
"DataSourceIntegrityIngestModule.process.verificationFailure={0} failed integrity verification",
})
@Override
public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
@ -276,13 +280,16 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
}
String verificationResultStr;
String messageResultStr;
MessageType messageType;
if (verified) {
messageType = MessageType.INFO;
verificationResultStr = NbBundle.getMessage(this.getClass(), "DataSourceIntegrityIngestModule.shutDown.verified");
messageResultStr = Bundle.DataSourceIntegrityIngestModule_process_verificationSuccess(imgName);
} else {
messageType = MessageType.WARNING;
verificationResultStr = NbBundle.getMessage(this.getClass(), "DataSourceIntegrityIngestModule.shutDown.notVerified");
messageResultStr = Bundle.DataSourceIntegrityIngestModule_process_verificationFailure(imgName);
}
detailedResults += NbBundle.getMessage(this.getClass(), "DataSourceIntegrityIngestModule.shutDown.resultLi", verificationResultStr);
@ -299,7 +306,7 @@ public class DataSourceIntegrityIngestModule implements DataSourceIngestModule {
}
services.postMessage(IngestMessage.createMessage(messageType, DataSourceIntegrityModuleFactory.getModuleName(),
imgName + verificationResultStr, detailedResults));
messageResultStr, detailedResults));
} else {
// Store the hashes in the database and update the image

89
Core/src/org/sleuthkit/autopsy/textextractors/ArtifactTextExtractor.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.textextractors;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Extracts text from artifacts by concatenating the values of all of the
* artifact's attributes.
*/
class ArtifactTextExtractor extends TextExtractor {
private final BlackboardArtifact artifact;
public ArtifactTextExtractor(Content artifact) {
this.artifact = (BlackboardArtifact) artifact;
}
@Override
public Reader getReader() throws ExtractionException {
// Concatenate the string values of all attributes into a single
// "content" string to be indexed.
StringBuilder artifactContents = new StringBuilder();
Content dataSource = null;
try {
dataSource = artifact.getDataSource();
} catch (TskCoreException tskCoreException) {
throw new ExtractionException("Unable to get datasource for artifact: " + artifact.toString(), tskCoreException);
}
if (dataSource == null) {
throw new ExtractionException("Datasource was null for artifact: " + artifact.toString());
}
try {
for (BlackboardAttribute attribute : artifact.getAttributes()) {
artifactContents.append(attribute.getAttributeType().getDisplayName());
artifactContents.append(" : ");
// We have also discussed modifying BlackboardAttribute.getDisplayString()
// to magically format datetime attributes but that is complicated by
// the fact that BlackboardAttribute exists in Sleuthkit data model
// while the utility to determine the timezone to use is in ContentUtils
// in the Autopsy datamodel.
switch (attribute.getValueType()) {
case DATETIME:
artifactContents.append(ContentUtils.getStringTime(attribute.getValueLong(), dataSource));
break;
default:
artifactContents.append(attribute.getDisplayString());
}
artifactContents.append(System.lineSeparator());
}
} catch (TskCoreException tskCoreException) {
throw new ExtractionException("Unable to get attributes for artifact: " + artifact.toString(), tskCoreException);
}
return new InputStreamReader(IOUtils.toInputStream(artifactContents,
StandardCharsets.UTF_8), StandardCharsets.UTF_8);
}
@Override
public boolean isSupported(Content file, String detectedFormat) {
return true;
}
}

2
Core/src/org/sleuthkit/autopsy/textextractors/Bundle.properties Executable file
View File

@ -0,0 +1,2 @@
AbstractFileTikaTextExtract.index.tikaParseTimeout.text=Exception\: Tika parse timeout for content\: {0}, {1}
AbstractFileTikaTextExtract.index.exception.tikaParse.msg=Exception\: Unexpected exception from Tika parse task execution for file\: {0}, {1}

2
Core/src/org/sleuthkit/autopsy/textextractors/Bundle_ja.properties Executable file
View File

@ -0,0 +1,2 @@
AbstractFileTikaTextExtract.index.exception.tikaParse.msg=\u4f8b\u5916\uff1a\u30d5\u30a1\u30a4\u30eb\uff1a{0}, {1}\u306eApache Tika\u30d1\u30fc\u30b9\u30bf\u30b9\u30af\u5b9f\u884c\u4e2d\u306e\u4e88\u671f\u305b\u306c\u4f8b\u5916
AbstractFileTikaTextExtract.index.tikaParseTimeout.text=\u4f8b\u5916\uff1a\u30b3\u30f3\u30c6\u30f3\u30c4\uff1a{0}, {1}\u306eApache Tika\u30d1\u30fc\u30b9\u306e\u30bf\u30a4\u30e0\u30a2\u30a6\u30c8

58
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HtmlTextExtractor.java → Core/src/org/sleuthkit/autopsy/textextractors/HtmlTextExtractor.java
View File

@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
package org.sleuthkit.autopsy.textextractors;
import java.io.IOException;
import java.io.Reader;
@ -38,10 +38,11 @@ import org.sleuthkit.datamodel.ReadContentInputStream;
/**
* Extracts text from HTML content.
*/
class HtmlTextExtractor extends ContentTextExtractor {
final class HtmlTextExtractor extends TextExtractor {
static final private Logger logger = Logger.getLogger(HtmlTextExtractor.class.getName());
private static final int MAX_SIZE = 50_000_000; //50MB
private final int MAX_SIZE;
private final Content file;
static final List<String> WEB_MIME_TYPES = Arrays.asList(
"application/javascript", //NON-NLS
@ -51,27 +52,51 @@ class HtmlTextExtractor extends ContentTextExtractor {
"text/html", //NON-NLS NON-NLS
"text/javascript" //NON-NLS
);
static {
// Disable Jericho HTML Parser log messages.
Config.LoggerProvider = LoggerProvider.DISABLED;
}
@Override
boolean isContentTypeSpecific() {
return true;
/**
* Creates a default instance of the HtmlTextExtractor. Supported file size
* is 50MB.
*/
public HtmlTextExtractor(Content file) {
//Set default to be 50 MB.
MAX_SIZE = 50_000_000;
this.file = file;
}
/**
* Determines if this content type is supported by this extractor.
*
* @param content Content instance to be analyzed
* @param detectedFormat Mimetype of content instance
*
* @return flag indicating support
*/
@Override
boolean isSupported(Content content, String detectedFormat) {
public boolean isSupported(Content content, String detectedFormat) {
return detectedFormat != null
&& WEB_MIME_TYPES.contains(detectedFormat)
&& content.getSize() <= MAX_SIZE;
}
/**
* Returns a reader that will iterate over the text of an HTML document.
*
* @param content Html document source
*
* @return A reader instance containing the document source text
*
* @throws TextExtractorException
*/
@Override
public Reader getReader(Content content) throws TextExtractorException {
ReadContentInputStream stream = new ReadContentInputStream(content);
public Reader getReader() throws ExtractionException {
//TODO JIRA-4467, there is only harm in excluding HTML documents greater
//than 50MB due to our troubled approach of extraction.
ReadContentInputStream stream = new ReadContentInputStream(file);
//Parse the stream with Jericho and put the results in a Reader
try {
@ -164,17 +189,8 @@ class HtmlTextExtractor extends ContentTextExtractor {
// All done, now make it a reader
return new StringReader(stringBuilder.toString());
} catch (IOException ex) {
throw new TextExtractorException("Error extracting HTML from content.", ex);
logger.log(Level.WARNING, "Error extracting HTML from content.", ex);
throw new ExtractionException("Error extracting HTML from content.", ex);
}
}
@Override
public boolean isDisabled() {
return false;
}
@Override
public void logWarning(final String msg, Exception ex) {
logger.log(Level.WARNING, msg, ex); //NON-NLS }
}
}

114
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java → Core/src/org/sleuthkit/autopsy/textextractors/SqliteTextExtractor.java
View File

@ -1,24 +1,23 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
/*
* Autopsy Forensic Browser
*
* Copyright 2018-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
package org.sleuthkit.autopsy.textextractors;
import com.google.common.io.CharSource;
import java.io.IOException;
import java.io.Reader;
import java.util.Iterator;
@ -28,37 +27,27 @@ import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.SQLiteTableReader;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
/**
* Dedicated SqliteTextExtractor to solve the problems associated with Tika's
* Sqlite parser.
* Extracts text from SQLite database files.
*
* Tika problems: 1) Tika fails to open virtual tables 2) Tika fails to open
* tables with spaces in table name 3) Tika fails to include the table names in
* output (except for the first table it parses)
* This is a dedicated solution to address the problems associated with
* Tika's sqlite parser (version 1.17), which include the following:
* 1) Virtual tables cause the parser to bail
* 2) Tables that contain spaces in their name are not extracted
* 3) Table names are not included in its output text
*/
class SqliteTextExtractor extends ContentTextExtractor {
final class SqliteTextExtractor extends TextExtractor {
private static final String SQLITE_MIMETYPE = "application/x-sqlite3";
private static final Logger logger = Logger.getLogger(SqliteTextExtractor.class.getName());
private final AbstractFile file;
@Override
boolean isContentTypeSpecific() {
return true;
public SqliteTextExtractor(Content file) {
this.file = (AbstractFile) file;
}
@Override
public boolean isDisabled() {
return false;
}
@Override
public void logWarning(String msg, Exception exception) {
logger.log(Level.WARNING, msg, exception); //NON-NLS
}
/**
* Supports only the sqlite mimetypes
*
@ -68,44 +57,34 @@ class SqliteTextExtractor extends ContentTextExtractor {
* @return true if x-sqlite3
*/
@Override
boolean isSupported(Content file, String detectedFormat) {
public boolean isSupported(Content file, String detectedFormat) {
return SQLITE_MIMETYPE.equals(detectedFormat);
}
/**
* Returns a stream that will read from a sqlite database.
* Returns a reader that will iterate over the text of a sqlite database.
*
* @param source Content file
*
* @return An InputStream that reads from a Sqlite database.
* @return An InputStream that reads from a Sqlite database
*
* @throws
* org.sleuthkit.autopsy.keywordsearch.TextExtractor.TextExtractorException
* @throws TextExtractorException
*/
@Override
public Reader getReader(Content source) throws TextExtractorException {
//Firewall for any content that is not an AbstractFile
if (!AbstractFile.class.isInstance(source)) {
try {
return CharSource.wrap("").openStream();
} catch (IOException ex) {
throw new TextExtractorException("", ex);
}
}
return new SQLiteStreamReader((AbstractFile) source);
public Reader getReader() throws ExtractionException {
return new SQLiteStreamReader(file);
}
/**
* Produces a continuous stream of characters from a database file. To
* achieve this, all table names are queues up and a SQLiteTableReader is
* used to do the actual queries and table iteration.
*/
public class SQLiteStreamReader extends Reader {
private class SQLiteStreamReader extends Reader {
private final SQLiteTableReader reader;
private final AbstractFile file;
private Iterator<String> tableNames;
private String currentTableName;
@ -217,9 +196,10 @@ class SqliteTextExtractor extends ContentTextExtractor {
}
/**
* Reads database values into the buffer. This function is responsible for
* getting the next table in the queue, initiating calls to the SQLiteTableReader,
* and filling in any excess bytes that are lingering from the previous call.
* Reads database values into the buffer. This function is responsible
* for getting the next table in the queue, initiating calls to the
* SQLiteTableReader, and filling in any excess bytes that are lingering
* from the previous call.
*
* @throws IOException
*/
@ -255,9 +235,9 @@ class SqliteTextExtractor extends ContentTextExtractor {
reader.read(currentTableName, () -> bufIndex == len);
} catch (SQLiteTableReaderException ex) {
logger.log(Level.WARNING, String.format(
"Error attempting to read file table: [%s]" //NON-NLS
+ " for file: [%s] (id=%d).", currentTableName, //NON-NLS
file.getName(), file.getId()), ex.getMessage());
"Error attempting to read file table: [%s]" //NON-NLS
+ " for file: [%s] (id=%d).", currentTableName, //NON-NLS
file.getName(), file.getId()), ex.getMessage());
}
} else {
if (bufIndex == off) {
@ -290,8 +270,8 @@ class SqliteTextExtractor extends ContentTextExtractor {
}
/**
* Wrapper that holds the excess bytes that were left over from the previous
* call to read().
* Wrapper that holds the excess bytes that were left over from the
* previous call to read().
*/
private class ExcessBytes {

153
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/StringsTextExtractor.java → Core/src/org/sleuthkit/autopsy/textextractors/StringsTextExtractor.java
View File

@ -16,19 +16,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
package org.sleuthkit.autopsy.textextractors;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import java.util.Objects;
import org.openide.util.Lookup;
import org.sleuthkit.autopsy.coreutils.StringExtract;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
import org.sleuthkit.autopsy.textextractors.extractionconfigs.DefaultExtractionConfig;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskException;
@ -36,24 +36,25 @@ import org.sleuthkit.datamodel.TskException;
/**
* Extracts raw strings from content.
*/
class StringsTextExtractor extends ContentTextExtractor {
final class StringsTextExtractor extends TextExtractor {
static final private Logger logger = Logger.getLogger(StringsTextExtractor.class.getName());
/**
* Options for this extractor
*/
enum ExtractOptions {
EXTRACT_UTF16, ///< extract UTF16 text, true/false
EXTRACT_UTF8, ///< extract UTF8 text, true/false
};
private boolean extractUTF8;
private boolean extractUTF16;
private final Content content;
private final static String DEFAULT_INDEXED_TEXT_CHARSET = "UTF-8";
private final List<SCRIPT> extractScripts = new ArrayList<>();
private Map<String, String> extractOptions = new HashMap<>();
public StringsTextExtractor() {
/**
* Creates a default StringsTextExtractor instance. The instance will be
* configured to run only LATIN_2 as its default extraction script and UTF-8
* as its default encoding.
*/
public StringsTextExtractor(Content content) {
//LATIN_2 is the default script
extractScripts.add(SCRIPT.LATIN_2);
extractUTF8 = true;
this.content = content;
}
/**
@ -61,56 +62,29 @@ class StringsTextExtractor extends ContentTextExtractor {
*
* @param extractScripts scripts to use
*/
public void setScripts(List<SCRIPT> extractScripts) {
public final void setScripts(List<SCRIPT> extractScripts) {
if (extractScripts == null) {
return;
}
this.extractScripts.clear();
this.extractScripts.addAll(extractScripts);
}
/**
* Get the currently used scripts for extraction
* Returns a reader that will iterate over the text of the content source.
*
* @return scripts currently used or null if not supported
*/
public List<SCRIPT> getScripts() {
return new ArrayList<>(extractScripts);
}
/**
* Get current options
* @param content Content source of any type
*
* @return currently used, extractor specific options, or null of not
* supported
*/
public Map<String, String> getOptions() {
return extractOptions;
}
/**
* Set extractor specific options
* @return A reader instance that content text can be obtained from
*
* @param options options to use
* @throws
* org.sleuthkit.autopsy.textextractors.TextExtractor.TextExtractorException
*/
public void setOptions(Map<String, String> options) {
this.extractOptions = options;
}
@Override
public void logWarning(final String msg, Exception ex) {
logger.log(Level.WARNING, msg, ex); //NON-NLS }
}
@Override
public boolean isDisabled() {
boolean extractUTF8 = Boolean.parseBoolean(extractOptions.get(ExtractOptions.EXTRACT_UTF8.toString()));
boolean extractUTF16 = Boolean.parseBoolean(extractOptions.get(ExtractOptions.EXTRACT_UTF16.toString()));
return extractUTF8 == false && extractUTF16 == false;
}
@Override
public InputStreamReader getReader(Content content) throws TextExtractorException {
public InputStreamReader getReader() {
InputStream stringStream = getInputStream(content);
return new InputStreamReader(stringStream, Server.DEFAULT_INDEXED_TEXT_CHARSET);
return new InputStreamReader(stringStream, Charset.forName(DEFAULT_INDEXED_TEXT_CHARSET));
}
InputStream getInputStream(Content content) {
@ -118,27 +92,55 @@ class StringsTextExtractor extends ContentTextExtractor {
if (extractScripts.size() == 1 && extractScripts.get(0).equals(SCRIPT.LATIN_1)) {
return new EnglishOnlyStream(content);//optimal for english, english only
} else {
boolean extractUTF8 = Boolean.parseBoolean(extractOptions.get(ExtractOptions.EXTRACT_UTF8.toString()));
boolean extractUTF16 = Boolean.parseBoolean(extractOptions.get(ExtractOptions.EXTRACT_UTF16.toString()));
return new InternationalStream(content, extractScripts, extractUTF8, extractUTF16);
}
}
/**
* Determines how the extraction process will proceed given the settings
* stored in this context instance.
*
* See the DefaultExtractionConfig class in the extractionconfigs package
* for available settings.
*
* @param context Lookup instance containing config classes
*/
@Override
public boolean isContentTypeSpecific() {
return false;
}
@Override
public boolean isSupported(Content content, String detectedFormat) {
// strings can be run on anything.
return true;
public void setExtractionSettings(Lookup context) {
if (context != null) {
DefaultExtractionConfig configInstance = context.lookup(DefaultExtractionConfig.class);
if (configInstance == null) {
return;
}
if (Objects.nonNull(configInstance.getExtractUTF8())) {
extractUTF8 = configInstance.getExtractUTF8();
}
if (Objects.nonNull(configInstance.getExtractUTF16())) {
extractUTF16 = configInstance.getExtractUTF16();
}
if (Objects.nonNull(configInstance.getExtractScripts())) {
setScripts(configInstance.getExtractScripts());
}
}
}
/**
* Content input string stream reader/converter - given Content,
* extract strings from it and return encoded bytes via read()
*
* @return
*/
@Override
public boolean isEnabled() {
return extractUTF8 || extractUTF16;
}
@Override
boolean isSupported(Content file, String detectedFormat) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
/**
* Content input string stream reader/converter - given Content, extract
* strings from it and return encoded bytes via read()
*
* Note: the utility supports extraction of only LATIN script and UTF8,
* UTF16LE, UTF16BE encodings and uses a brute force encoding detection -
@ -150,7 +152,6 @@ class StringsTextExtractor extends ContentTextExtractor {
*/
private static class EnglishOnlyStream extends InputStream {
private static final Logger logger = Logger.getLogger(EnglishOnlyStream.class.getName());
private static final String NLS = Character.toString((char) 10); //new line
private static final int READ_BUF_SIZE = 65536;
private static final int MIN_PRINTABLE_CHARS = 4; //num. of chars needed to qualify as a char string
@ -244,12 +245,7 @@ class StringsTextExtractor extends ContentTextExtractor {
}
//get char from cur read buf
char c = (char) curReadBuf[readBufOffset++];
if (c == 0 && singleConsecZero == false) {
//preserve the current sequence if max consec. 1 zero char
singleConsecZero = true;
} else {
singleConsecZero = false;
}
singleConsecZero = c == 0 && singleConsecZero == false; //preserve the current sequence if max consec. 1 zero char
if (StringExtract.isPrintableAscii(c)) {
tempString.append(c);
++tempStringLen;
@ -328,7 +324,7 @@ class StringsTextExtractor extends ContentTextExtractor {
private int copyToReturn(byte[] b, int off, long len) {
final String curStringS = curString.toString();
//logger.log(Level.INFO, curStringS);
byte[] stringBytes = curStringS.getBytes(Server.DEFAULT_INDEXED_TEXT_CHARSET);
byte[] stringBytes = curStringS.getBytes(Charset.forName(DEFAULT_INDEXED_TEXT_CHARSET));
System.arraycopy(stringBytes, 0, b, off, Math.min(curStringLen, (int) len));
//logger.log(Level.INFO, curStringS);
//copied all string, reset
@ -370,7 +366,6 @@ class StringsTextExtractor extends ContentTextExtractor {
*/
private static class InternationalStream extends InputStream {
private static final Logger logger = Logger.getLogger(InternationalStream.class.getName());
private static final int FILE_BUF_SIZE = 1024 * 1024;
private final Content content;
private final byte[] oneCharBuf = new byte[1];
@ -499,7 +494,7 @@ class StringsTextExtractor extends ContentTextExtractor {
*/
private void convert(int numBytes) {
lastExtractResult = stringExtractor.extract(fileReadBuff, numBytes, 0);
convertBuff = lastExtractResult.getText().getBytes(Server.DEFAULT_INDEXED_TEXT_CHARSET);
convertBuff = lastExtractResult.getText().getBytes(Charset.forName(DEFAULT_INDEXED_TEXT_CHARSET));
//reset tracking vars
if (lastExtractResult.getNumBytes() == 0) {
bytesInConvertBuff = 0;

103
Core/src/org/sleuthkit/autopsy/textextractors/TextExtractor.java Normal file
View File

@ -0,0 +1,103 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-18 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.textextractors;
import java.io.Reader;
import org.openide.util.Lookup;
import org.sleuthkit.datamodel.Content;
/**
* Extracts the text out of {@link org.sleuthkit.datamodel.Content} instances
* and exposes them as a {@link java.io.Reader}. Concrete implementations can be
* obtained from
* {@link org.sleuthkit.autopsy.textextractors.TextExtractorFactory#getExtractor(org.sleuthkit.datamodel.Content)}
* or
* {@link org.sleuthkit.autopsy.textextractors.TextExtractorFactory#getExtractor(org.sleuthkit.datamodel.Content, org.openide.util.Lookup)}.
*
* @see org.sleuthkit.autopsy.textextractors.TextExtractorFactory
*/
public abstract class TextExtractor {
/**
* Determines if the file content is supported by the extractor.
*
* @param file to test if its content should be supported
* @param detectedFormat mime-type with detected format (such as text/plain)
* or null if not detected
*
* @return true if the file content is supported, false otherwise
*/
abstract boolean isSupported(Content file, String detectedFormat);
/**
* Determines if the TextExtractor instance is enabled to read content.
*
* @return
*/
boolean isEnabled() {
return true;
}
/**
* Get a {@link java.io.Reader} that will iterate over the text extracted
* from the {@link org.sleuthkit.datamodel.Content} passed into
* {@link org.sleuthkit.autopsy.textextractors.TextExtractorFactory}.
*
* @return {@link java.io.Reader} that contains the text of the underlying
* {@link org.sleuthkit.datamodel.Content}
*
* @throws
* org.sleuthkit.autopsy.textextractors.TextExtractor.ExtractionException
*
* @see org.sleuthkit.autopsy.textextractors.TextExtractorFactory
*
*/
public abstract Reader getReader() throws ExtractionException;
/**
* Determines how the extraction process will proceed given the settings
* stored in the context instance.
*
* @param context Instance containing file config classes
*/
void setExtractionSettings(Lookup context) {
//no-op by default
}
/**
* Exception encountered during
* {@link org.sleuthkit.autopsy.textextractors.TextExtractor#getReader()}.
* This indicates that there was an internal parsing error that occurred
* during the reading of Content text.
*/
public class ExtractionException extends Exception {
public ExtractionException(String msg, Throwable ex) {
super(msg, ex);
}
public ExtractionException(Throwable ex) {
super(ex);
}
public ExtractionException(String msg) {
super(msg);
}
}
}

157
Core/src/org/sleuthkit/autopsy/textextractors/TextExtractorFactory.java Executable file
View File

@ -0,0 +1,157 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.textextractors;
import java.util.Arrays;
import java.util.List;
import org.openide.util.Lookup;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.Report;
/**
* Factory for creating
* {@link org.sleuthkit.autopsy.textextractors.TextExtractor}'s given a
* {@link org.sleuthkit.datamodel.Content} instance
*
* See {@link org.sleuthkit.autopsy.textextractors.extractionconfigs} for
* available {@link org.sleuthkit.autopsy.textextractors.TextExtractor}
* configuration options.
*
* @see org.openide.util.Lookup
*/
public class TextExtractorFactory {
/**
* Auto detects the correct
* {@link org.sleuthkit.autopsy.textextractors.TextExtractor} given the
* {@link org.sleuthkit.datamodel.Content}.
*
* See {@link org.sleuthkit.autopsy.textextractors.extractionconfigs} for
* available {@link org.sleuthkit.autopsy.textextractors.TextExtractor}
* configuration options.
*
* @param content Content source that will be read from
* @param context Contains extraction configurations for certain file types
*
* @return A TextExtractor that supports the given content. File text can be
* obtained from
* {@link org.sleuthkit.autopsy.textextractors.TextExtractor#getReader()}.
*
* @throws NoTextExtractorFound Encountered when there is no TextExtractor
* was found for the given content type. Use {@link
* TextExtractorFactory#getDefaultExtractor(org.sleuthkit.datamodel.Content,
* org.openide.util.Lookup)}
*
* @see org.openide.util.Lookup
*/
public static TextExtractor getExtractor(Content content,
Lookup context) throws NoTextExtractorFound {
if (content instanceof AbstractFile) {
String mimeType = ((AbstractFile) content).getMIMEType();
List<TextExtractor> extractors = Arrays.asList(
new HtmlTextExtractor(content),
new SqliteTextExtractor(content),
new TikaTextExtractor(content));
for (TextExtractor extractor : extractors) {
extractor.setExtractionSettings(context);
if (extractor.isEnabled() && extractor.isSupported(content, mimeType)) {
return extractor;
}
}
} else if (content instanceof BlackboardArtifact) {
TextExtractor artifactExtractor = new ArtifactTextExtractor((BlackboardArtifact) content);
artifactExtractor.setExtractionSettings(context);
return artifactExtractor;
} else if (content instanceof Report) {
TextExtractor reportExtractor = new TikaTextExtractor(content);
reportExtractor.setExtractionSettings(context);
return reportExtractor;
}
throw new NoTextExtractorFound(
String.format("Could not find a suitable extractor for "
+ "content with name [%s] and id=[%d]. Try using the default, "
+ "non content specific extractor as an alternative.",
content.getName(), content.getId())
);
}
/**
* Auto detects the correct
* {@link org.sleuthkit.autopsy.textextractors.TextExtractor} given the
* {@link org.sleuthkit.datamodel.Content}.
*
* @param content Content instance that will be read from
*
* @return A TextExtractor that supports the given content. File text can be
* obtained from {@link TextExtractor#getReader()}.
*
* @throws NoTextExtractorFound Encountered when there is no TextExtractor
* was found for the given content type. Use {@link
* TextExtractorFactory#getDefaultExtractor(org.sleuthkit.datamodel.Content,
* org.openide.util.Lookup)}
*/
public static TextExtractor getExtractor(Content content)
throws NoTextExtractorFound {
return getExtractor(content, null);
}
/**
* Returns the default extractor that can be run on any content type. This
* extractor should be used as a backup in the event that no extractor was
* found using or {@link TextExtractorFactory#getDefaultExtractor(org.sleuthkit.datamodel.Content, org.openide.util.Lookup)}
* {@link TextExtractorFactory#getExtractor(org.sleuthkit.datamodel.Content)}.
*
* @param content Content source to read from
* @param context Contains extraction configurations for certain file types
*
* @return A DefaultExtractor instance. File text can be obtained from
* {@link TextExtractor#getReader()}.
*
* @see org.openide.util.Lookup
*/
public static TextExtractor getDefaultExtractor(Content content, Lookup context) {
TextExtractor stringsInstance = new StringsTextExtractor(content);
stringsInstance.setExtractionSettings(context);
return stringsInstance;
}
/**
* System level exception for handling content types that have no specific
* strategy defined for extracting their text.
*
* @see
* org.sleuthkit.autopsy.textextractors.TextExtractorFactory#getExtractor(org.sleuthkit.datamodel.Content)
* @see
* org.sleuthkit.autopsy.textextractors.TextExtractorFactory#getDefaultExtractor(org.sleuthkit.datamodel.Content,
* org.openide.util.Lookup)}
*/
public static class NoTextExtractorFound extends Exception {
public NoTextExtractorFound(String msg) {
super(msg);
}
public NoTextExtractorFound(Throwable ex) {
super(ex);
}
}
}

187
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TikaTextExtractor.java → Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java
View File

@ -16,15 +16,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
package org.sleuthkit.autopsy.textextractors;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharSource;
import java.io.File;
import java.io.IOException;
import java.io.PushbackReader;
import java.io.Reader;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@ -33,6 +37,7 @@ import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FilenameUtils;
import org.apache.tika.Tika;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
@ -44,26 +49,78 @@ import org.apache.tika.parser.ocr.TesseractOCRConfig;
import org.apache.tika.parser.pdf.PDFParserConfig;
import org.openide.util.NbBundle;
import org.openide.modules.InstalledFileLocator;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.openide.util.Lookup;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.textextractors.extractionconfigs.ImageFileExtractionConfig;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ReadContentInputStream;
/**
* Extracts text from Tika supported content. Protects against Tika
* parser hangs (for unexpected/corrupt content) using a timeout mechanism.
* Extracts text from Tika supported content. Protects against Tika parser hangs
* (for unexpected/corrupt content) using a timeout mechanism.
*/
class TikaTextExtractor extends ContentTextExtractor {
final class TikaTextExtractor extends TextExtractor {
//Mimetype groups to aassist extractor implementations in ignoring binary and
//archive files.
private static final List<String> BINARY_MIME_TYPES
= ImmutableList.of(
//ignore binary blob data, for which string extraction will be used
"application/octet-stream", //NON-NLS
"application/x-msdownload"); //NON-NLS
/**
* generally text extractors should ignore archives and let unpacking
* modules take care of them
*/
private static final List<String> ARCHIVE_MIME_TYPES
= ImmutableList.of(
//ignore unstructured binary and compressed data, for which string extraction or unzipper works better
"application/x-7z-compressed", //NON-NLS
"application/x-ace-compressed", //NON-NLS
"application/x-alz-compressed", //NON-NLS
"application/x-arj", //NON-NLS
"application/vnd.ms-cab-compressed", //NON-NLS
"application/x-cfs-compressed", //NON-NLS
"application/x-dgc-compressed", //NON-NLS
"application/x-apple-diskimage", //NON-NLS
"application/x-gca-compressed", //NON-NLS
"application/x-dar", //NON-NLS
"application/x-lzx", //NON-NLS
"application/x-lzh", //NON-NLS
"application/x-rar-compressed", //NON-NLS
"application/x-stuffit", //NON-NLS
"application/x-stuffitx", //NON-NLS
"application/x-gtar", //NON-NLS
"application/x-archive", //NON-NLS
"application/x-executable", //NON-NLS
"application/x-gzip", //NON-NLS
"application/zip", //NON-NLS
"application/x-zoo", //NON-NLS
"application/x-cpio", //NON-NLS
"application/x-shar", //NON-NLS
"application/x-tar", //NON-NLS
"application/x-bzip", //NON-NLS
"application/x-bzip2", //NON-NLS
"application/x-lzip", //NON-NLS
"application/x-lzma", //NON-NLS
"application/x-lzop", //NON-NLS
"application/x-z", //NON-NLS
"application/x-compress"); //NON-NLS
private static final java.util.logging.Logger tikaLogger = java.util.logging.Logger.getLogger("Tika"); //NON-NLS
static final private Logger logger = Logger.getLogger(TikaTextExtractor.class.getName());
private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor();
private static final String SQLITE_MIMETYPE = "application/x-sqlite3";
private final AutoDetectParser parser = new AutoDetectParser();
private final Content content;
private boolean tesseractOCREnabled;
private static final String TESSERACT_DIR_NAME = "Tesseract-OCR"; //NON-NLS
private static final String TESSERACT_EXECUTABLE = "tesseract.exe"; //NON-NLS
private static final File TESSERACT_PATH = locateTesseractExecutable();
private static final String LANGUAGE_PACKS = getLanguagePacks();
private static final List<String> TIKA_SUPPORTED_TYPES
= new Tika().getParser().getSupportedTypes(new ParseContext())
@ -71,13 +128,23 @@ class TikaTextExtractor extends ContentTextExtractor {
.map(mt -> mt.getType() + "/" + mt.getSubtype())
.collect(Collectors.toList());
@Override
public void logWarning(final String msg, Exception ex) {
KeywordSearch.getTikaLogger().log(Level.WARNING, msg, ex);
public TikaTextExtractor(Content content) {
this.content = content;
}
/**
* Returns a reader that will iterate over the text extracted from Apache
* Tika.
*
* @param content Supported source content to extract
*
* @return Reader that contains Apache Tika extracted text
*
* @throws
* org.sleuthkit.autopsy.textextractors.TextExtractor.TextExtractorException
*/
@Override
public Reader getReader(Content content) throws TextExtractorException {
public Reader getReader() throws ExtractionException {
ReadContentInputStream stream = new ReadContentInputStream(content);
Metadata metadata = new Metadata();
@ -90,28 +157,28 @@ class TikaTextExtractor extends ContentTextExtractor {
officeParserConfig.setUseSAXPptxExtractor(true);
officeParserConfig.setUseSAXDocxExtractor(true);
parseContext.set(OfficeParserConfig.class, officeParserConfig);
// configure OCR if it is enabled in KWS settings and installed on the machine
if (TESSERACT_PATH != null && KeywordSearchSettings.getOcrOption() && PlatformUtil.isWindowsOS() == true) {
if (TESSERACT_PATH != null && tesseractOCREnabled && PlatformUtil.isWindowsOS() == true) {
// configure PDFParser.
PDFParserConfig pdfConfig = new PDFParserConfig();
// Extracting the inline images and letting Tesseract run on each inline image.
// https://wiki.apache.org/tika/PDFParser%20%28Apache%20PDFBox%29
// https://tika.apache.org/1.7/api/org/apache/tika/parser/pdf/PDFParserConfig.html
pdfConfig.setExtractInlineImages(true);
pdfConfig.setExtractInlineImages(true);
// Multiple pages within a PDF file might refer to the same underlying image.
pdfConfig.setExtractUniqueInlineImagesOnly(true);
pdfConfig.setExtractUniqueInlineImagesOnly(true);
parseContext.set(PDFParserConfig.class, pdfConfig);
// Configure Tesseract parser to perform OCR
TesseractOCRConfig ocrConfig = new TesseractOCRConfig();
String tesseractFolder = TESSERACT_PATH.getParent();
ocrConfig.setTesseractPath(tesseractFolder);
// Tesseract expects language data packs to be in a subdirectory of tesseractFolder, in a folder called "tessdata".
// If they are stored somewhere else, use ocrConfig.setTessdataPath(String tessdataPath) to point to them
ocrConfig.setLanguage("eng");
ocrConfig.setLanguage(LANGUAGE_PACKS);
parseContext.set(TesseractOCRConfig.class, ocrConfig);
}
@ -124,7 +191,7 @@ class TikaTextExtractor extends ContentTextExtractor {
PushbackReader pushbackReader = new PushbackReader(tikaReader);
int read = pushbackReader.read();
if (read == -1) {
throw new TextExtractorException("Unable to extract text: Tika returned empty reader for " + content);
throw new ExtractionException("Unable to extract text: Tika returned empty reader for " + content);
}
pushbackReader.unread(read);
@ -133,15 +200,13 @@ class TikaTextExtractor extends ContentTextExtractor {
return CharSource.concat(new ReaderCharSource(pushbackReader), metaDataCharSource).openStream();
} catch (TimeoutException te) {
final String msg = NbBundle.getMessage(this.getClass(), "AbstractFileTikaTextExtract.index.tikaParseTimeout.text", content.getId(), content.getName());
logWarning(msg, te);
throw new TextExtractorException(msg, te);
} catch (TextExtractorException ex) {
throw new ExtractionException(msg, te);
} catch (ExtractionException ex) {
throw ex;
} catch (Exception ex) {
KeywordSearch.getTikaLogger().log(Level.WARNING, "Exception: Unable to Tika parse the content" + content.getId() + ": " + content.getName(), ex.getCause()); //NON-NLS
tikaLogger.log(Level.WARNING, "Exception: Unable to Tika parse the content" + content.getId() + ": " + content.getName(), ex.getCause()); //NON-NLS
final String msg = NbBundle.getMessage(this.getClass(), "AbstractFileTikaTextExtract.index.exception.tikaParse.msg", content.getId(), content.getName());
logWarning(msg, ex);
throw new TextExtractorException(msg, ex);
throw new ExtractionException(msg, ex);
} finally {
future.cancel(true);
}
@ -187,16 +252,19 @@ class TikaTextExtractor extends ContentTextExtractor {
));
}
@Override
public boolean isContentTypeSpecific() {
return true;
}
/**
* Determines if Tika is supported for this content type and mimetype.
*
* @param content Source content to read
* @param detectedFormat Mimetype of content
*
* @return Flag indicating support for reading content type
*/
@Override
public boolean isSupported(Content content, String detectedFormat) {
if (detectedFormat == null
|| ContentTextExtractor.BINARY_MIME_TYPES.contains(detectedFormat) //any binary unstructured blobs (string extraction will be used)
|| ContentTextExtractor.ARCHIVE_MIME_TYPES.contains(detectedFormat)
|| BINARY_MIME_TYPES.contains(detectedFormat) //any binary unstructured blobs (string extraction will be used)
|| ARCHIVE_MIME_TYPES.contains(detectedFormat)
|| (detectedFormat.startsWith("video/") && !detectedFormat.equals("video/x-flv")) //skip video other than flv (tika supports flv only) //NON-NLS
|| detectedFormat.equals(SQLITE_MIMETYPE) //Skip sqlite files, Tika cannot handle virtual tables and will fail with an exception. //NON-NLS
) {
@ -205,9 +273,34 @@ class TikaTextExtractor extends ContentTextExtractor {
return TIKA_SUPPORTED_TYPES.contains(detectedFormat);
}
@Override
public boolean isDisabled() {
return false;
/**
* Retrieves all of the installed language packs from their designated
* directory location to be used to configure Tesseract OCR.
*
* @return String of all language packs available for Tesseract to use
*/
private static String getLanguagePacks() {
File languagePackRootDir = new File(TESSERACT_PATH.getParent(), "tessdata");
//Acceptable extensions for Tesseract-OCR version 3.05 language packs.
//All extensions other than traineddata are associated with cube files that
//have been made obsolete since version 4.0.
List<String> acceptableExtensions = Arrays.asList("traineddata", "params",
"lm", "fold", "bigrams", "nn", "word-freq", "size",
"user-patterns", "user-words");
//Pull out only unique languagePacks
HashSet<String> languagePacks = new HashSet<>();
if (languagePackRootDir.exists()) {
for (File languagePack : languagePackRootDir.listFiles()) {
if (languagePack.isDirectory() || !acceptableExtensions.contains(
FilenameUtils.getExtension(languagePack.getName()))) {
continue;
}
String threeLetterPackageName = languagePack.getName().substring(0, 3);
//Ignore the eng language pack if accidentally added
languagePacks.add(threeLetterPackageName);
}
}
return String.join("+", languagePacks);
}
/**
@ -233,6 +326,28 @@ class TikaTextExtractor extends ContentTextExtractor {
}
/**
* Determines how the extraction process will proceed given the settings
* stored in this context instance.
*
* See the ImageFileExtractionConfig class in the extractionconfigs package
* for available settings.
*
* @param context Instance containing config classes
*/
@Override
public void setExtractionSettings(Lookup context) {
if (context != null) {
ImageFileExtractionConfig configInstance = context.lookup(ImageFileExtractionConfig.class);
if (configInstance == null) {
return;
}
if (Objects.nonNull(configInstance.getOCREnabled())) {
this.tesseractOCREnabled = configInstance.getOCREnabled();
}
}
}
/**
* An implementation of CharSource that just wraps an existing reader and
* returns it in openStream().

100
Core/src/org/sleuthkit/autopsy/textextractors/extractionconfigs/DefaultExtractionConfig.java Executable file
View File

@ -0,0 +1,100 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.textextractors.extractionconfigs;
import java.util.List;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
/**
* Allows for configuration of the
* {@link org.sleuthkit.autopsy.textextractors.TextExtractor} obtained from
* {@link org.sleuthkit.autopsy.textextractors.TextExtractorFactory#getDefaultExtractor(org.sleuthkit.datamodel.Content, org.openide.util.Lookup)}.
*
* The default extractor will read strings from the Content instance. This class
* allows for the configuration of the encoding language script to use during
* extraction.
*
* @see org.sleuthkit.autopsy.textextractors.TextExtractorFactory
* @see
* org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT
* @see org.openide.util.Lookup
*/
public class DefaultExtractionConfig {
private Boolean extractUTF8;
private Boolean extractUTF16;
private List<SCRIPT> extractScripts;
/**
* Enables UTF-8 encoding to be used during extraction.
*
* @param enabled Flag indicating if UTF-8 should be turned on
*/
public void setExtractUTF8(boolean enabled) {
this.extractUTF8 = enabled;
}
/**
* Enables UTF-16 encoding to be used during extraction.
*
* @param enabled Flag indicating if UTF-16 should be turned on
*/
public void setExtractUTF16(boolean enabled) {
this.extractUTF16 = enabled;
}
/**
* Returns whether extracting with UTF-8 encoding should be done.
*
* @return Flag indicating if UTF-8 has been turned on/off
*/
public Boolean getExtractUTF8() {
return extractUTF8;
}
/**
* Return whether extracting with UTF-16 encoding should be done.
*
* @return Flag indicating if UTF-16 has been turned on/off
*/
public Boolean getExtractUTF16() {
return extractUTF16;
}
/**
* Sets the type of extraction scripts that will be used during this
* extraction. See
* {@link org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT}
* for more information about available scripts.
*
* @param scripts Desired set of scripts to be used during extraction
*/
public void setExtractScripts(List<SCRIPT> scripts) {
this.extractScripts = scripts;
}
/**
* Gets the desired set of scripts to be used during extraction.
*
* @return Set of extraction scripts to be used
*/
public List<SCRIPT> getExtractScripts() {
return this.extractScripts;
}
}

54
Core/src/org/sleuthkit/autopsy/textextractors/extractionconfigs/ImageFileExtractionConfig.java Executable file
View File

@ -0,0 +1,54 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.textextractors.extractionconfigs;
/**
* Allows for configuration of OCR on image files.
* {@link org.sleuthkit.autopsy.textextractors.TextExtractor}'s that use
* ImageFileExtractionConfig can be obtained through
* {@link org.sleuthkit.autopsy.textextractors.TextExtractorFactory#getExtractor(org.sleuthkit.datamodel.Content)}
* or
* {@link org.sleuthkit.autopsy.textextractors.TextExtractorFactory#getDefaultExtractor(org.sleuthkit.datamodel.Content, org.openide.util.Lookup)}.
*
* @see org.sleuthkit.autopsy.textextractors.TextExtractorFactory
* @see org.openide.util.Lookup
*/
public class ImageFileExtractionConfig {
private Boolean OCREnabled;
/**
* Enables OCR to be run on the text extractor responsible for handling
* image files.
*
* @param enabled Flag indicating if OCR is enabled.
*/
public void setOCREnabled(boolean enabled) {
this.OCREnabled = enabled;
}
/**
* Gets the OCR flag that has been set. By default this flag is turned off.
*
* @return Flag indicating if OCR is enabled.
*/
public boolean getOCREnabled() {
return this.OCREnabled;
}
}

2
Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java
View File

@ -19,7 +19,7 @@
package org.sleuthkit.autopsy.texttranslation;
/**
* Provides a system exception for the Text Translation errors
* Provides a system exception for Text Translation errors
*/
public class TranslationException extends Exception {

12
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/CommonAttributeSearchInterCaseTests.java
View File

@ -31,7 +31,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.AllInterCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import static org.sleuthkit.autopsy.commonfilessearch.InterCaseTestUtils.CASE1;
import static org.sleuthkit.autopsy.commonfilessearch.InterCaseTestUtils.CASE2;
import static org.sleuthkit.autopsy.commonfilessearch.InterCaseTestUtils.CASE3;
@ -118,7 +118,7 @@ public class CommonAttributeSearchInterCaseTests extends NbTestCase {
AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(false, false, type, 0);
CommonAttributeSearchResults metadata = builder.findMatches();
CommonAttributeCountSearchResults metadata = builder.findMatchesByCount();
metadata.size();
@ -151,20 +151,20 @@ public class CommonAttributeSearchInterCaseTests extends NbTestCase {
// try {
//
// AbstractCommonAttributeSearcher builder;
// CommonAttributeSearchResults metadata;
// CommonAttributeCountSearchResults metadata;
//
// builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.USB_ID_TYPE, 100);
// metadata = builder.findMatches();
// metadata = builder.findMatchesByCount();
// metadata.size();
// //assertTrue("This should yield 13 results.", verifyInstanceCount(metadata, 13));
//
// builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.USB_ID_TYPE, 20);
// metadata = builder.findMatches();
// metadata = builder.findMatchesByCount();
// metadata.size();
// //assertTrue("This should yield no results.", verifyInstanceCount(metadata, 0));
//
// builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.USB_ID_TYPE, 90);
// metadata = builder.findMatches();
// metadata = builder.findMatchesByCount();
// metadata.size();
// //assertTrue("This should yield 2 results.", verifyInstanceCount(metadata, 2));
//

8
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithHashAndFileTypeInterCaseTests.java
View File

@ -31,7 +31,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.AllInterCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.SingleInterCaseCommonAttributeSearcher;
import static org.sleuthkit.autopsy.commonfilessearch.InterCaseTestUtils.*;
import org.sleuthkit.datamodel.TskCoreException;
@ -103,7 +103,7 @@ public class IngestedWithHashAndFileTypeInterCaseTests extends NbTestCase {
// try {
// //note that the params false and false are presently meaningless because that feature is not supported yet
// AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.FILE_TYPE, 0);
// CommonAttributeSearchResults metadata = builder.findMatches();
// CommonAttributeCountSearchResults metadata = builder.findMatches();
//
// assertTrue("Results should not be empty", metadata.size() != 0);
//
@ -154,7 +154,7 @@ public class IngestedWithHashAndFileTypeInterCaseTests extends NbTestCase {
// CorrelationAttributeInstance.Type fileType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0);
// AbstractCommonAttributeSearcher builder = new SingleInterCaseCommonAttributeSearcher(matchesMustAlsoBeFoundInThisCase, false, false, fileType, 0);
//
// CommonAttributeSearchResults metadata = builder.findMatches();
// CommonAttributeCountSearchResults metadata = builder.findMatches();
//
// assertTrue("Results should not be empty", metadata.size() != 0);
//
@ -206,7 +206,7 @@ public class IngestedWithHashAndFileTypeInterCaseTests extends NbTestCase {
// CorrelationAttributeInstance.Type fileType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0);
// AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(false, false, fileType, 50);
//
// CommonAttributeSearchResults metadata = builder.findMatches();
// CommonAttributeCountSearchResults metadata = builder.findMatches();
//
// assertTrue("Results should not be empty", metadata.size() != 0);
//

20
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithHashAndFileTypeIntraCaseTests.java
View File

@ -33,7 +33,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.AllIntraCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.SingleIntraCaseCommonAttributeSearcher;
import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseTestUtils.*;
import org.sleuthkit.autopsy.ingest.IngestJobSettings;
@ -100,7 +100,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Map<Long, String> dataSources = this.utils.getDataSourceMap();
//
// AbstractCommonAttributeSearcher allSourcesBuilder = new AllIntraCaseCommonAttributeSearcher(dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = allSourcesBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = allSourcesBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = IntraCaseTestUtils.mapFileInstancesToDataSources(metadata);
//
@ -141,7 +141,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Map<Long, String> dataSources = this.utils.getDataSourceMap();
//
// AbstractCommonAttributeSearcher allSourcesBuilder = new AllIntraCaseCommonAttributeSearcher(dataSources, true, false, 0);
// CommonAttributeSearchResults metadata = allSourcesBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = allSourcesBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//
@ -182,7 +182,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Map<Long, String> dataSources = this.utils.getDataSourceMap();
//
// AbstractCommonAttributeSearcher allSourcesBuilder = new AllIntraCaseCommonAttributeSearcher(dataSources, false, true, 0);
// CommonAttributeSearchResults metadata = allSourcesBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = allSourcesBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//
@ -224,7 +224,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Long first = getDataSourceIdByName(SET1, dataSources);
//
// AbstractCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(first, dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//
@ -266,7 +266,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Long first = getDataSourceIdByName(SET1, dataSources);
//
// AbstractCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(first, dataSources, true, false, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//
@ -308,7 +308,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Long first = getDataSourceIdByName(SET1, dataSources);
//
// AbstractCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(first, dataSources, false, true, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//
@ -350,7 +350,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Long second = getDataSourceIdByName(SET2, dataSources);
//
// AbstractCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(second, dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//
@ -391,7 +391,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Long last = getDataSourceIdByName(SET4, dataSources);
//
// AbstractCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(last, dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//
@ -432,7 +432,7 @@ public class IngestedWithHashAndFileTypeIntraCaseTests extends NbTestCase {
// Long third = getDataSourceIdByName(SET3, dataSources);
//
// AbstractCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(third, dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = mapFileInstancesToDataSources(metadata);
//

6
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithNoFileTypesIntraCaseTests.java
View File

@ -32,7 +32,7 @@ import junit.framework.Assert;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.commonfilesearch.AllIntraCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.IntraCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.SingleIntraCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.ingest.IngestJobSettings;
@ -102,7 +102,7 @@ public class IngestedWithNoFileTypesIntraCaseTests extends NbTestCase {
// Map<Long, String> dataSources = this.utils.getDataSourceMap();
//
// IntraCaseCommonAttributeSearcher allSourcesBuilder = new AllIntraCaseCommonAttributeSearcher(dataSources, true, false, 0);
// CommonAttributeSearchResults metadata = allSourcesBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = allSourcesBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = IntraCaseTestUtils.mapFileInstancesToDataSources(metadata);
//
@ -126,7 +126,7 @@ public class IngestedWithNoFileTypesIntraCaseTests extends NbTestCase {
// Long third = IntraCaseTestUtils.getDataSourceIdByName(IntraCaseTestUtils.SET3, dataSources);
//
// IntraCaseCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(third, dataSources, true, false, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = IntraCaseTestUtils.mapFileInstancesToDataSources(metadata);
//

32
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/InterCaseTestUtils.java
View File

@ -55,7 +55,7 @@ import org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeInstance;
import org.sleuthkit.autopsy.commonfilesearch.CaseDBCommonAttributeInstanceNode;
import org.sleuthkit.autopsy.commonfilesearch.CentralRepoCommonAttributeInstance;
import org.sleuthkit.autopsy.commonfilesearch.CentralRepoCommonAttributeInstanceNode;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import org.sleuthkit.autopsy.guiutils.DataSourceLoader;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeValue;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeValueList;
@ -411,10 +411,8 @@ class InterCaseTestUtils {
}
}
static boolean verifyInstanceCount(CommonAttributeSearchResults searchDomain, int instanceCount) {
try {
static boolean verifyInstanceCount(CommonAttributeCountSearchResults searchDomain, int instanceCount) {
int tally = 0;
for (Map.Entry<Integer, CommonAttributeValueList> entry : searchDomain.getMetadata().entrySet()) {
entry.getValue().displayDelayedMetadata();
for (CommonAttributeValue value : entry.getValue().getMetadataList()) {
@ -422,21 +420,11 @@ class InterCaseTestUtils {
tally += value.getInstanceCount();
}
}
return tally == instanceCount;
} catch (EamDbException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex.getMessage());
return false;
}
}
static boolean verifyInstanceExistenceAndCount(CommonAttributeSearchResults searchDomain, String fileName, String dataSource, String crCase, int instanceCount) {
try {
static boolean verifyInstanceExistenceAndCount(CommonAttributeCountSearchResults searchDomain, String fileName, String dataSource, String crCase, int instanceCount) {
int tally = 0;
for (Map.Entry<Integer, CommonAttributeValueList> entry : searchDomain.getMetadata().entrySet()) {
entry.getValue().displayDelayedMetadata();
for (CommonAttributeValue value : entry.getValue().getMetadataList()) {
@ -494,14 +482,7 @@ class InterCaseTestUtils {
}
}
}
return tally == instanceCount;
} catch (EamDbException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex.getMessage());
return false;
}
}
/**
@ -520,8 +501,7 @@ class InterCaseTestUtils {
*
* @return true if yes, else false
*/
boolean areAllResultsOfType(CommonAttributeSearchResults metadata, CorrelationAttributeInstance.Type attributeType) {
try {
boolean areAllResultsOfType(CommonAttributeCountSearchResults metadata, CorrelationAttributeInstance.Type attributeType) {
for (CommonAttributeValueList matches : metadata.getMetadata().values()) {
for (CommonAttributeValue value : matches.getMetadataList()) {
return value
@ -532,9 +512,5 @@ class InterCaseTestUtils {
return false;
}
return false;
} catch (EamDbException ex) {
Assert.fail(ex.getMessage());
return false;
}
}
}

14
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseTestUtils.java
View File

@ -18,7 +18,6 @@
*/
package org.sleuthkit.autopsy.commonfilessearch;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.SQLException;
@ -33,9 +32,8 @@ import junit.framework.Assert;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.ImageDSProcessor;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeInstance;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import org.sleuthkit.autopsy.guiutils.DataSourceLoader;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeValue;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeValueList;
@ -189,10 +187,8 @@ class IntraCaseTestUtils {
* @param metadata object returned by the code under test
* @return mapping of objectId to data source name
*/
static Map<Long, String> mapFileInstancesToDataSources(CommonAttributeSearchResults metadata) {
static Map<Long, String> mapFileInstancesToDataSources(CommonAttributeCountSearchResults metadata) {
Map<Long, String> instanceIdToDataSource = new HashMap<>();
try {
for (Map.Entry<Integer, CommonAttributeValueList> entry : metadata.getMetadata().entrySet()) {
entry.getValue().displayDelayedMetadata();
for (CommonAttributeValue md : entry.getValue().getMetadataList()) {
@ -201,13 +197,7 @@ class IntraCaseTestUtils {
}
}
}
return instanceIdToDataSource;
} catch (EamDbException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex.getMessage());
return instanceIdToDataSource;
}
}
static List<AbstractFile> getFiles(Set<Long> objectIds) {

4
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSourcesIntraCaseTests.java
View File

@ -34,7 +34,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException;
import org.sleuthkit.autopsy.commonfilesearch.AbstractCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.AllIntraCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseTestUtils.*;
import org.sleuthkit.autopsy.ingest.IngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestModuleTemplate;
@ -106,7 +106,7 @@ public class MatchesInAtLeastTwoSourcesIntraCaseTests extends NbTestCase {
// Map<Long, String> dataSources = this.utils.getDataSourceMap();
//
// AbstractCommonAttributeSearcher allSourcesBuilder = new AllIntraCaseCommonAttributeSearcher(dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = allSourcesBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = allSourcesBuilder.findMatches();
//
// Map<Long, String> objectIdToDataSource = IntraCaseTestUtils.mapFileInstancesToDataSources(metadata);
//

6
Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/UningestedCasesIntraCaseTests.java
View File

@ -29,7 +29,7 @@ import org.openide.util.Exceptions;
import junit.framework.Assert;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.commonfilesearch.AllIntraCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.CommonAttributeCountSearchResults;
import org.sleuthkit.autopsy.commonfilesearch.IntraCaseCommonAttributeSearcher;
import org.sleuthkit.autopsy.commonfilesearch.SingleIntraCaseCommonAttributeSearcher;
import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseTestUtils.SET1;
@ -81,7 +81,7 @@ public class UningestedCasesIntraCaseTests extends NbTestCase {
// Map<Long, String> dataSources = this.utils.getDataSourceMap();
//
// IntraCaseCommonAttributeSearcher allSourcesBuilder = new AllIntraCaseCommonAttributeSearcher(dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = allSourcesBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = allSourcesBuilder.findMatches();
//
// int resultCount = metadata.size();
// assertEquals(resultCount, 0);
@ -101,7 +101,7 @@ public class UningestedCasesIntraCaseTests extends NbTestCase {
// Long first = getDataSourceIdByName(SET1, dataSources);
//
// IntraCaseCommonAttributeSearcher singleSourceBuilder = new SingleIntraCaseCommonAttributeSearcher(first, dataSources, false, false, 0);
// CommonAttributeSearchResults metadata = singleSourceBuilder.findMatches();
// CommonAttributeCountSearchResults metadata = singleSourceBuilder.findMatches();
//
// int resultCount = metadata.size();
// assertEquals(resultCount, 0);

150
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ArtifactTextExtractor.java
View File

@ -1,150 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.util.logging.Level;
import org.apache.commons.io.IOUtils;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Extracts text from artifacts by concatenating the values of all of the
* artifact's attributes.
*/
class ArtifactTextExtractor implements TextExtractor<BlackboardArtifact> {
static final private Logger logger = Logger.getLogger(ArtifactTextExtractor.class.getName());
/**
* Get the Content that is the data source for the given artifact. //JMTODO:
* is there a prexisting method to do this?
*
* @param artifact
*
* @return The data source for the given artifact as a Content object, or
* null if it could not be found.
*
* @throws TskCoreException if there is a problem accessing the case db.
*/
static Content getDataSource(BlackboardArtifact artifact) throws TskCoreException {
Case currentCase;
try {
currentCase = Case.getCurrentCaseThrows();
} catch (NoCurrentCaseException ignore) {
// thorown by Case.getCurrentOpenCase() if currentCase is null
return null;
}
SleuthkitCase sleuthkitCase = currentCase.getSleuthkitCase();
if (sleuthkitCase == null) {
return null;
}
Content dataSource;
AbstractFile abstractFile = sleuthkitCase.getAbstractFileById(artifact.getObjectID());
if (abstractFile != null) {
dataSource = abstractFile.getDataSource();
} else {
dataSource = sleuthkitCase.getContentById(artifact.getObjectID());
}
if (dataSource == null) {
return null;
}
return dataSource;
}
@Override
public boolean isDisabled() {
return false;
}
@Override
public void logWarning(final String msg, Exception ex) {
logger.log(Level.WARNING, msg, ex); //NON-NLS }
}
private InputStream getInputStream(BlackboardArtifact artifact) throws TextExtractorException {
// Concatenate the string values of all attributes into a single
// "content" string to be indexed.
StringBuilder artifactContents = new StringBuilder();
Content dataSource = null;
try {
dataSource = getDataSource(artifact);
} catch (TskCoreException tskCoreException) {
throw new TextExtractorException("Unable to get datasource for artifact: " + artifact.toString(), tskCoreException);
}
if (dataSource == null) {
throw new TextExtractorException("Datasource was null for artifact: " + artifact.toString());
}
try {
for (BlackboardAttribute attribute : artifact.getAttributes()) {
artifactContents.append(attribute.getAttributeType().getDisplayName());
artifactContents.append(" : ");
// We have also discussed modifying BlackboardAttribute.getDisplayString()
// to magically format datetime attributes but that is complicated by
// the fact that BlackboardAttribute exists in Sleuthkit data model
// while the utility to determine the timezone to use is in ContentUtils
// in the Autopsy datamodel.
switch (attribute.getValueType()) {
case DATETIME:
artifactContents.append(ContentUtils.getStringTime(attribute.getValueLong(), dataSource));
break;
default:
artifactContents.append(attribute.getDisplayString());
}
artifactContents.append(System.lineSeparator());
}
} catch (TskCoreException tskCoreException) {
throw new TextExtractorException("Unable to get attributes for artifact: " + artifact.toString(), tskCoreException);
}
return IOUtils.toInputStream(artifactContents, StandardCharsets.UTF_8);
}
@Override
public Reader getReader(BlackboardArtifact source) throws TextExtractorException {
return new InputStreamReader(getInputStream(source), StandardCharsets.UTF_8);
}
@Override
public long getID(BlackboardArtifact source) {
return source.getArtifactID();
}
@Override
public String getName(BlackboardArtifact source) {
return source.getDisplayName() + "_" + source.getArtifactID();
}
}

110
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ContentTextExtractor.java
View File

@ -1,110 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.io.Reader;
import java.util.Arrays;
import java.util.List;
import org.sleuthkit.datamodel.Content;
/**
* Common methods for utilities that extract text and content and divide into
* chunks
*/
abstract class ContentTextExtractor implements TextExtractor<Content> {
static final List<String> BINARY_MIME_TYPES
= Arrays.asList(
//ignore binary blob data, for which string extraction will be used
"application/octet-stream", //NON-NLS
"application/x-msdownload"); //NON-NLS
/** generally text extractors should ignore archives and let unpacking
* modules take care of them */
static final List<String> ARCHIVE_MIME_TYPES
= Arrays.asList(
//ignore unstructured binary and compressed data, for which string extraction or unzipper works better
"application/x-7z-compressed", //NON-NLS
"application/x-ace-compressed", //NON-NLS
"application/x-alz-compressed", //NON-NLS
"application/x-arj", //NON-NLS
"application/vnd.ms-cab-compressed", //NON-NLS
"application/x-cfs-compressed", //NON-NLS
"application/x-dgc-compressed", //NON-NLS
"application/x-apple-diskimage", //NON-NLS
"application/x-gca-compressed", //NON-NLS
"application/x-dar", //NON-NLS
"application/x-lzx", //NON-NLS
"application/x-lzh", //NON-NLS
"application/x-rar-compressed", //NON-NLS
"application/x-stuffit", //NON-NLS
"application/x-stuffitx", //NON-NLS
"application/x-gtar", //NON-NLS
"application/x-archive", //NON-NLS
"application/x-executable", //NON-NLS
"application/x-gzip", //NON-NLS
"application/zip", //NON-NLS
"application/x-zoo", //NON-NLS
"application/x-cpio", //NON-NLS
"application/x-shar", //NON-NLS
"application/x-tar", //NON-NLS
"application/x-bzip", //NON-NLS
"application/x-bzip2", //NON-NLS
"application/x-lzip", //NON-NLS
"application/x-lzma", //NON-NLS
"application/x-lzop", //NON-NLS
"application/x-z", //NON-NLS
"application/x-compress"); //NON-NLS
/**
* Determines if the extractor works only for specified types is
* supportedTypes() or whether is a generic content extractor (such as
* string extractor)
*
* @return
*/
abstract boolean isContentTypeSpecific();
/**
* Determines if the file content is supported by the extractor if
* isContentTypeSpecific() returns true.
*
* @param content to test if its content should be supported
* @param detectedFormat mime-type with detected format (such as text/plain)
* or null if not detected
*
* @return true if the file content is supported, false otherwise
*/
abstract boolean isSupported(Content file, String detectedFormat);
@Override
public abstract Reader getReader(Content source) throws TextExtractorException;
@Override
public long getID(Content source) {
return source.getId();
}
@Override
public String getName(Content source) {
return source.getName();
}
}

31
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.keywordsearch;
import java.io.BufferedReader;
import java.io.Reader;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
@ -58,7 +59,6 @@ class Ingester {
private final Server solrServer = KeywordSearch.getServer();
private static final SolrFieldsVisitor SOLR_FIELDS_VISITOR = new SolrFieldsVisitor();
private static Ingester instance;
private static final int SINGLE_READ_CHARS = 512;
private Ingester() {
}
@ -106,8 +106,8 @@ class Ingester {
* @throws IngesterException if there was an error processing a specific
* artifact, but the Solr server is probably fine.
*/
void indexMetaDataOnly(BlackboardArtifact artifact) throws IngesterException {
indexChunk("", new ArtifactTextExtractor().getName(artifact), getContentFields(artifact));
void indexMetaDataOnly(BlackboardArtifact artifact, String sourceName) throws IngesterException {
indexChunk("", sourceName, getContentFields(artifact));
}
/**
@ -142,23 +142,12 @@ class Ingester {
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
*/
// TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients
< T extends SleuthkitVisitableItem> boolean indexText(TextExtractor< T> extractor, T source, IngestJobContext context) throws Ingester.IngesterException {
final long sourceID = extractor.getID(source);
final String sourceName = extractor.getName(source);
< T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException {
int numChunks = 0; //unknown until chunking is done
if (extractor.isDisabled()) {
/*
* some Extractors, notable the strings extractor, have options
* which can be configured such that no extraction should be done
*/
return true;
}
Map<String, String> fields = getContentFields(source);
//Get a reader for the content of the given source
try (BufferedReader reader = new BufferedReader(extractor.getReader(source));) {
try (BufferedReader reader = new BufferedReader(sourceReader)) {
Chunker chunker = new Chunker(reader);
for (Chunk chunk : chunker) {
if (context != null && context.fileIngestIsCancelled()) {
@ -173,18 +162,18 @@ class Ingester {
indexChunk(chunk.toString(), sourceName, fields);
numChunks++;
} catch (Ingester.IngesterException ingEx) {
extractor.logWarning("Ingester had a problem with extracted string from file '" //NON-NLS
logger.log(Level.WARNING, "Ingester had a problem with extracted string from file '" //NON-NLS
+ sourceName + "' (id: " + sourceID + ").", ingEx);//NON-NLS
throw ingEx; //need to rethrow to signal error and move on
}
}
if (chunker.hasException()) {
extractor.logWarning("Error chunking content from " + sourceID + ": " + sourceName, chunker.getException());
logger.log(Level.WARNING, "Error chunking content from " + sourceID + ": " + sourceName, chunker.getException());
return false;
}
} catch (Exception ex) {
extractor.logWarning("Unexpected error, can't read content stream from " + sourceID + ": " + sourceName, ex);//NON-NLS
logger.log(Level.WARNING, "Unexpected error, can't read content stream from " + sourceID + ": " + sourceName, ex);//NON-NLS
return false;
} finally {
if (context != null && context.fileIngestIsCancelled()) {
@ -371,7 +360,7 @@ class Ingester {
Map<String, String> params = new HashMap<>();
params.put(Server.Schema.ID.toString(), Long.toString(artifact.getArtifactID()));
try {
params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(ArtifactTextExtractor.getDataSource(artifact).getId()));
params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(artifact.getDataSource().getId()));
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Could not get data source id to properly index the artifact " + artifact.getArtifactID(), ex); //NON-NLS
params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1));

15
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalLanguageSettingsPanel.java
View File

@ -35,6 +35,7 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.StringExtract;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.StringsExtractOptions;
/**
* Child panel of the global settings panel (Languages tab).
@ -45,7 +46,7 @@ class KeywordSearchGlobalLanguageSettingsPanel extends javax.swing.JPanel implem
private final Map<String, StringExtract.StringExtractUnicodeTable.SCRIPT> scripts = new HashMap<>();
private ActionListener updateLanguagesAction;
private List<SCRIPT> toUpdate;
KeywordSearchGlobalLanguageSettingsPanel() {
initComponents();
customizeComponents();
@ -125,12 +126,12 @@ class KeywordSearchGlobalLanguageSettingsPanel extends javax.swing.JPanel implem
private void reloadScriptsCheckBoxes() {
boolean utf16
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF16.toString()));
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsExtractOptions.EXTRACT_UTF16.toString()));
enableUTF16Checkbox.setSelected(utf16);
boolean utf8
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF8.toString()));
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsExtractOptions.EXTRACT_UTF8.toString()));
enableUTF8Checkbox.setSelected(utf8);
boolean ocr = KeywordSearchSettings.getOcrOption();
@ -152,12 +153,12 @@ class KeywordSearchGlobalLanguageSettingsPanel extends javax.swing.JPanel implem
reloadScriptsCheckBoxes();
boolean utf16
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF16.toString()));
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsExtractOptions.EXTRACT_UTF16.toString()));
enableUTF16Checkbox.setSelected(utf16);
boolean utf8
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF8.toString()));
= Boolean.parseBoolean(KeywordSearchSettings.getStringExtractOption(StringsExtractOptions.EXTRACT_UTF8.toString()));
enableUTF8Checkbox.setSelected(utf8);
final boolean extractEnabled = utf16 || utf8;
@ -316,9 +317,9 @@ class KeywordSearchGlobalLanguageSettingsPanel extends javax.swing.JPanel implem
@Override
public void store() {
KeywordSearchSettings.setStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF8.toString(),
KeywordSearchSettings.setStringExtractOption(StringsExtractOptions.EXTRACT_UTF8.toString(),
Boolean.toString(enableUTF8Checkbox.isSelected()));
KeywordSearchSettings.setStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF16.toString(),
KeywordSearchSettings.setStringExtractOption(StringsExtractOptions.EXTRACT_UTF16.toString(),
Boolean.toString(enableUTF16Checkbox.isSelected()));
KeywordSearchSettings.setOcrOption(enableOcrCheckbox.isSelected());

133
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -18,14 +18,18 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.util.ArrayList;
import com.google.common.collect.ImmutableList;
import java.io.Reader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import org.openide.util.Exceptions;
import org.openide.util.Lookup;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.openide.util.lookup.Lookups;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.Logger;
@ -37,9 +41,15 @@ import org.sleuthkit.autopsy.ingest.IngestMessage.MessageType;
import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException;
import org.sleuthkit.autopsy.keywordsearch.TextFileExtractor.TextFileExtractorException;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
import org.sleuthkit.autopsy.textextractors.TextExtractor;
import org.sleuthkit.autopsy.textextractors.TextExtractor.ExtractionException;
import org.sleuthkit.autopsy.textextractors.TextExtractorFactory;
import org.sleuthkit.autopsy.textextractors.extractionconfigs.ImageFileExtractionConfig;
import org.sleuthkit.autopsy.textextractors.extractionconfigs.DefaultExtractionConfig;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.FileKnown;
@ -61,6 +71,52 @@ import org.sleuthkit.datamodel.TskData.FileKnown;
"CannotRunFileTypeDetection=Unable to run file type detection."
})
public final class KeywordSearchIngestModule implements FileIngestModule {
/** generally text extractors should ignore archives and let unpacking
* modules take care of them */
public static final List<String> ARCHIVE_MIME_TYPES
= ImmutableList.of(
//ignore unstructured binary and compressed data, for which string extraction or unzipper works better
"application/x-7z-compressed", //NON-NLS
"application/x-ace-compressed", //NON-NLS
"application/x-alz-compressed", //NON-NLS
"application/x-arj", //NON-NLS
"application/vnd.ms-cab-compressed", //NON-NLS
"application/x-cfs-compressed", //NON-NLS
"application/x-dgc-compressed", //NON-NLS
"application/x-apple-diskimage", //NON-NLS
"application/x-gca-compressed", //NON-NLS
"application/x-dar", //NON-NLS
"application/x-lzx", //NON-NLS
"application/x-lzh", //NON-NLS
"application/x-rar-compressed", //NON-NLS
"application/x-stuffit", //NON-NLS
"application/x-stuffitx", //NON-NLS
"application/x-gtar", //NON-NLS
"application/x-archive", //NON-NLS
"application/x-executable", //NON-NLS
"application/x-gzip", //NON-NLS
"application/zip", //NON-NLS
"application/x-zoo", //NON-NLS
"application/x-cpio", //NON-NLS
"application/x-shar", //NON-NLS
"application/x-tar", //NON-NLS
"application/x-bzip", //NON-NLS
"application/x-bzip2", //NON-NLS
"application/x-lzip", //NON-NLS
"application/x-lzma", //NON-NLS
"application/x-lzop", //NON-NLS
"application/x-z", //NON-NLS
"application/x-compress"); //NON-NLS
/**
* Options for this extractor
*/
enum StringsExtractOptions {
EXTRACT_UTF16, ///< extract UTF16 text, true/false
EXTRACT_UTF8, ///< extract UTF8 text, true/false
};
enum UpdateFrequency {
@ -89,13 +145,10 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
//accessed read-only by searcher thread
private boolean startedSearching = false;
private List<ContentTextExtractor> textExtractors;
private StringsTextExtractor stringExtractor;
private TextFileExtractor txtFileExtractor;
private Lookup stringsExtractionContext;
private final KeywordSearchJobSettings settings;
private boolean initialized = false;
private long jobId;
private long dataSourceId;
private static final AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging
private int instanceNum = 0;
private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
@ -152,7 +205,6 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
public void startUp(IngestJobContext context) throws IngestModuleException {
initialized = false;
jobId = context.getJobId();
dataSourceId = context.getDataSource().getId();
Server server = KeywordSearch.getServer();
if (server.coreIsOpen() == false) {
@ -238,22 +290,15 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
}
}
//initialize extractors
stringExtractor = new StringsTextExtractor();
stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts());
stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions());
txtFileExtractor = new TextFileExtractor();
textExtractors = new ArrayList<>();
//order matters, more specific extractors first
textExtractors.add(new HtmlTextExtractor());
//Add sqlite text extractor to be default for sqlite files, since tika stuggles
//with them. See SqliteTextExtractor class for specifics
textExtractors.add(new SqliteTextExtractor());
textExtractors.add(new TikaTextExtractor());
DefaultExtractionConfig stringsConfig = new DefaultExtractionConfig();
Map<String, String> stringsOptions = KeywordSearchSettings.getStringExtractOptions();
stringsConfig.setExtractUTF8(Boolean.parseBoolean(stringsOptions.get(StringsExtractOptions.EXTRACT_UTF8.toString())));
stringsConfig.setExtractUTF16(Boolean.parseBoolean(stringsOptions.get(StringsExtractOptions.EXTRACT_UTF16.toString())));
stringsConfig.setExtractScripts(KeywordSearchSettings.getStringExtractScripts());
stringsExtractionContext = Lookups.fixed(stringsConfig);
indexer = new Indexer();
initialized = true;
}
@ -345,10 +390,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
* Common cleanup code when module stops or final searcher completes
*/
private void cleanup() {
textExtractors.clear();
textExtractors = null;
stringExtractor = null;
txtFileExtractor = null;
stringsExtractionContext = null;
initialized = false;
}
@ -436,24 +478,18 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
* @throws IngesterException exception thrown if indexing failed
*/
private boolean extractTextAndIndex(AbstractFile aFile, String detectedFormat) throws IngesterException {
ContentTextExtractor extractor = null;
//go over available text extractors in order, and pick the first one (most specific one)
for (ContentTextExtractor fe : textExtractors) {
if (fe.isSupported(aFile, detectedFormat)) {
extractor = fe;
break;
}
}
if (extractor == null) {
// No text extractor found.
ImageFileExtractionConfig imageConfig = new ImageFileExtractionConfig();
imageConfig.setOCREnabled(KeywordSearchSettings.getOcrOption());
Lookup extractionContext = Lookups.fixed(imageConfig);
try {
Reader specializedReader = TextExtractorFactory.getExtractor(aFile,extractionContext).getReader();
//divide into chunks and index
return Ingester.getDefault().indexText(specializedReader,aFile.getId(),aFile.getName(), aFile, context);
} catch (TextExtractorFactory.NoTextExtractorFound | ExtractionException ex) {
//No text extractor found... run the default instead
return false;
}
//logger.log(Level.INFO, "Extractor: " + fileExtract + ", file: " + aFile.getName());
//divide into chunks and index
return Ingester.getDefault().indexText(extractor, aFile, context);
}
/**
@ -469,7 +505,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return true;
}
if (Ingester.getDefault().indexText(stringExtractor, aFile, KeywordSearchIngestModule.this.context)) {
Reader stringsReader = TextExtractorFactory.getDefaultExtractor(aFile, stringsExtractionContext).getReader();
if (Ingester.getDefault().indexText(stringsReader,aFile.getId(),aFile.getName(), aFile, KeywordSearchIngestModule.this.context)) {
putIngestStatus(jobId, aFile.getId(), IngestStatus.STRINGS_INGESTED);
return true;
} else {
@ -477,7 +514,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
return false;
}
} catch (IngesterException ex) {
} catch (IngesterException | ExtractionException ex) {
logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").", ex); //NON-NLS
putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
return false;
@ -529,7 +566,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
// we skip archive formats that are opened by the archive module.
// @@@ We could have a check here to see if the archive module was enabled though...
if (ContentTextExtractor.ARCHIVE_MIME_TYPES.contains(fileType)) {
if (ARCHIVE_MIME_TYPES.contains(fileType)) {
try {
if (context.fileIngestIsCancelled()) {
return;
@ -577,11 +614,13 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
//Carved Files should be the only type of unallocated files capable of a txt extension and
//should be ignored by the TextFileExtractor because they may contain more than one text encoding
try {
if (Ingester.getDefault().indexText(txtFileExtractor, aFile, context)) {
TextFileExtractor textFileExtractor = new TextFileExtractor();
Reader textReader = textFileExtractor.getReader(aFile);
if (Ingester.getDefault().indexText(textReader, aFile.getId(), aFile.getName(), aFile, context)) {
putIngestStatus(jobId, aFile.getId(), IngestStatus.TEXT_INGESTED);
wasTextAdded = true;
}
} catch (IngesterException ex) {
} catch (IngesterException | TextFileExtractorException ex) {
logger.log(Level.WARNING, "Unable to index as unicode", ex);
}
}

5
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.java
View File

@ -31,6 +31,7 @@ import javax.swing.table.TableColumn;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.StringsExtractOptions;
/**
* Ingest job settings panel for keyword search file ingest modules.
@ -102,8 +103,8 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
}
private void displayEncodings() {
String utf8 = KeywordSearchSettings.getStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF8.toString());
String utf16 = KeywordSearchSettings.getStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF16.toString());
String utf8 = KeywordSearchSettings.getStringExtractOption(StringsExtractOptions.EXTRACT_UTF8.toString());
String utf16 = KeywordSearchSettings.getStringExtractOption(StringsExtractOptions.EXTRACT_UTF16.toString());
ArrayList<String> encodingsList = new ArrayList<>();
if (utf8 == null || Boolean.parseBoolean(utf8)) {
encodingsList.add("UTF8");

9
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java
View File

@ -28,6 +28,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.coreutils.StringExtract;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.StringsExtractOptions;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.UpdateFrequency;
//This file contains constants and settings for KeywordSearch
@ -234,14 +235,14 @@ class KeywordSearchSettings {
KeywordSearchSettings.setUpdateFrequency(UpdateFrequency.DEFAULT);
}
//setting default Extract UTF8
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, StringsTextExtractor.ExtractOptions.EXTRACT_UTF8.toString())) {
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, StringsExtractOptions.EXTRACT_UTF8.toString())) {
logger.log(Level.INFO, "No configuration for UTF8 found, generating default..."); //NON-NLS
KeywordSearchSettings.setStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF8.toString(), Boolean.TRUE.toString());
KeywordSearchSettings.setStringExtractOption(StringsExtractOptions.EXTRACT_UTF8.toString(), Boolean.TRUE.toString());
}
//setting default Extract UTF16
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, StringsTextExtractor.ExtractOptions.EXTRACT_UTF16.toString())) {
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, StringsExtractOptions.EXTRACT_UTF16.toString())) {
logger.log(Level.INFO, "No configuration for UTF16 found, generating defaults..."); //NON-NLS
KeywordSearchSettings.setStringExtractOption(StringsTextExtractor.ExtractOptions.EXTRACT_UTF16.toString(), Boolean.TRUE.toString());
KeywordSearchSettings.setStringExtractOption(StringsExtractOptions.EXTRACT_UTF16.toString(), Boolean.TRUE.toString());
}
//setting OCR default (disabled by default)
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, OCR_ENABLED)) {

32
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearch;
import com.google.common.eventbus.Subscribe;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
import java.util.ArrayList;
@ -47,10 +48,11 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
import org.sleuthkit.autopsy.progress.ProgressIndicator;
import org.sleuthkit.autopsy.textextractors.TextExtractor.ExtractionException;
import org.sleuthkit.autopsy.textextractors.TextExtractorFactory;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/**
@ -116,19 +118,24 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
return;
}
try {
ingester.indexMetaDataOnly(artifact);
ingester.indexText(new ArtifactTextExtractor(), artifact, null);
} catch (Ingester.IngesterException ex) {
Reader blackboardReader = TextExtractorFactory
.getExtractor(content, null).getReader();
String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
ingester.indexMetaDataOnly(artifact, sourceName);
ingester.indexText(blackboardReader, artifact.getArtifactID(), sourceName, content, null);
} catch (Ingester.IngesterException | TextExtractorFactory.NoTextExtractorFound | ExtractionException ex) {
throw new TskCoreException(ex.getCause().getMessage(), ex);
}
} else {
try {
ingester.indexText(new TikaTextExtractor(), content, null);
} catch (Ingester.IngesterException ex) {
Reader contentReader = TextExtractorFactory
.getExtractor(content, null).getReader();
ingester.indexText(contentReader, content.getId(), content.getName(), content, null);
} catch (TextExtractorFactory.NoTextExtractorFound | ExtractionException | Ingester.IngesterException ex) {
try {
// Try the StringsTextExtractor if Tika extractions fails.
ingester.indexText(new StringsTextExtractor(), content, null);
} catch (Ingester.IngesterException ex1) {
ingester.indexText(TextExtractorFactory.getDefaultExtractor(content, null).getReader(), content.getId(), content.getName(), content, null);
} catch (Ingester.IngesterException | ExtractionException ex1) {
throw new TskCoreException(ex.getCause().getMessage(), ex1);
}
}
@ -469,9 +476,12 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
final Ingester ingester = Ingester.getDefault();
try {
ingester.indexMetaDataOnly(artifact);
ingester.indexText(new ArtifactTextExtractor(), artifact, null);
} catch (Ingester.IngesterException ex) {
String sourceName = artifact.getDisplayName() + "_" + artifact.getArtifactID();
Reader contentSpecificReader
= TextExtractorFactory.getExtractor((Content) artifact, null).getReader();
ingester.indexMetaDataOnly(artifact, sourceName);
ingester.indexText(contentSpecificReader, artifact.getId(), sourceName, artifact, null);
} catch (Ingester.IngesterException | TextExtractorFactory.NoTextExtractorFound | ExtractionException ex) {
throw new TskCoreException(ex.getCause().getMessage(), ex);
}
}

88
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextExtractor.java
View File

@ -1,88 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-16 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.io.Reader;
import org.sleuthkit.datamodel.SleuthkitVisitableItem;
/**
* Extracts text out of a SleuthkitVisitableItem, and exposes it is a Reader.
* This Reader is given to the Ingester to chunk and index in Solr.
*
* @param <TextSource> The subtype of SleuthkitVisitableItem an implementation
* is able to process.
*/
interface TextExtractor< TextSource extends SleuthkitVisitableItem> {
/**
* Is this extractor configured such that no extraction will/should be done?
*
* @return True if this extractor will/should not perform any extraction.
*/
abstract boolean isDisabled();
/**
* Log the given message and exception as a warning.
*
* @param msg
* @param ex
*/
abstract void logWarning(String msg, Exception ex);
/**
* Get a reader that over the text extracted from the given source.
*
* @param stream
* @param source
*
* @return
*
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
*/
abstract Reader getReader(TextSource source) throws TextExtractorException;
/**
* Get the 'object' id of the given source.
*
* @param source
*
* @return
*/
abstract long getID(TextSource source);
/**
* Get a human readable name for the given source.
*
* @param source
*
* @return
*/
abstract String getName(TextSource source);
class TextExtractorException extends Exception {
public TextExtractorException(String message) {
super(message);
}
public TextExtractorException(String message, Throwable cause) {
super(message, cause);
}
}
}

42
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java
View File

@ -21,17 +21,15 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.BufferedInputStream;
import java.io.Reader;
import java.util.logging.Level;
import org.apache.tika.parser.txt.CharsetDetector;
import org.apache.tika.parser.txt.CharsetMatch;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.ReadContentInputStream;
/**
* Extract text from .txt files
*/
final class TextFileExtractor extends ContentTextExtractor {
final class TextFileExtractor {
//Set a Minimum confidence value to reject matches that may not have a valid text encoding
//Values of valid text encodings were generally 100, xml code sometimes had a value around 50,
@ -39,44 +37,30 @@ final class TextFileExtractor extends ContentTextExtractor {
//This limited information was used to select the current value as one that would filter out clearly non-text
//files while hopefully working on all files with a valid text encoding
static final private int MIN_MATCH_CONFIDENCE = 20;
static final private Logger logger = Logger.getLogger(TextFileExtractor.class.getName());
@Override
boolean isContentTypeSpecific() {
return true;
}
@Override
boolean isSupported(Content file, String detectedFormat) {
return true;
}
@Override
public Reader getReader(Content source) throws TextExtractorException {
public Reader getReader(AbstractFile source) throws TextFileExtractorException {
CharsetDetector detector = new CharsetDetector();
//wrap stream in a BufferedInputStream so that it supports the mark/reset methods necessary for the CharsetDetector
InputStream stream = new BufferedInputStream(new ReadContentInputStream(source));
try {
detector.setText(stream);
} catch (IOException ex) {
throw new TextExtractorException("Unable to get string from detected text in TextFileExtractor", ex);
throw new TextFileExtractorException("Unable to get string from detected text in TextFileExtractor", ex);
}
CharsetMatch match = detector.detect();
if (match.getConfidence() < MIN_MATCH_CONFIDENCE) {
throw new TextExtractorException("Text does not match any character set with a high enough confidence for TextFileExtractor");
throw new TextFileExtractorException("Text does not match any character set with a high enough confidence for TextFileExtractor");
}
return match.getReader();
}
@Override
public boolean isDisabled() {
return false;
public class TextFileExtractorException extends Exception {
public TextFileExtractorException(String msg, Throwable ex) {
super(msg, ex);
}
public TextFileExtractorException(String msg) {
super(msg);
}
}
@Override
public void logWarning(String msg, Exception ex) {
logger.log(Level.WARNING, msg, ex);
}
}

2
branding/core/core.jar/org/netbeans/core/startup/Bundle.properties
View File

@ -1,5 +1,5 @@
#Updated by build script
#Thu, 29 Nov 2018 12:23:03 -0500
#Tue, 11 Dec 2018 14:41:40 -0500
LBL_splash_window_title=Starting Autopsy
SPLASH_HEIGHT=314
SPLASH_WIDTH=538

2
branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties
View File

@ -1,4 +1,4 @@
#Updated by build script
#Thu, 29 Nov 2018 12:23:03 -0500
#Tue, 11 Dec 2018 14:41:40 -0500
CTL_MainWindow_Title=Autopsy 4.9.1
CTL_MainWindow_Title_No_Project=Autopsy 4.9.1