From fc51baa55479b8384e1bda8980e6b74a47ff5f08 Mon Sep 17 00:00:00 2001 From: esaunders Date: Mon, 29 Jan 2018 16:19:32 -0500 Subject: [PATCH 1/2] Added support for Report indexing. --- .../sleuthkit/autopsy/casemodule/Case.java | 47 +- .../KeywordSearchService.java | 12 +- KeywordSearch/arch.xml | 1076 +++++++++++++++++ .../autopsy/keywordsearch/Ingester.java | 29 +- .../keywordsearch/SolrSearchService.java | 32 +- .../keywordsearch/TikaTextExtractor.java | 1 - .../recentactivity/ExtractRegistry.java | 30 +- 7 files changed, 1202 insertions(+), 25 deletions(-) create mode 100644 KeywordSearch/arch.xml diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 804e224a70..dc2cc7f659 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -998,17 +998,16 @@ public class Case { } } - private static String getNameForTitle(){ + private static String getNameForTitle() { //Method should become unnecessary once technical debt story 3334 is done. - if (UserPreferences.getAppName().equals(Version.getName())){ + if (UserPreferences.getAppName().equals(Version.getName())) { //Available version number is version number for this application return String.format("%s %s", UserPreferences.getAppName(), Version.getVersion()); - } - else { + } else { return UserPreferences.getAppName(); } } - + /** * Update the GUI to to reflect the current case. */ @@ -1526,7 +1525,25 @@ public class Case { * @throws TskCoreException if there is a problem adding the report to the * case database. */ - public void addReport(String localPath, String srcModuleName, String reportName) throws TskCoreException { + public Report addReport(String localPath, String srcModuleName, String reportName) throws TskCoreException { + return addReport(localPath, srcModuleName, reportName, null); + } + + /** + * Adds a report to the case. + * + * @param localPath The path of the report file, must be in the case + * directory or one of its subdirectories. + * @param srcModuleName The name of the module that created the report. + * @param reportName The report name, may be empty. + * @param source The Content used to create the report, may be empty. + * + * @return The new Report instance. + * + * @throws TskCoreException if there is a problem adding the report to the + * case database. + */ + public Report addReport(String localPath, String srcModuleName, String reportName, Content source) throws TskCoreException { String normalizedLocalPath; try { normalizedLocalPath = Paths.get(localPath).normalize().toString(); @@ -1536,6 +1553,7 @@ public class Case { } Report report = this.caseDb.addReport(normalizedLocalPath, srcModuleName, reportName); eventPublisher.publish(new ReportAddedEvent(report)); + return report; } /** @@ -1616,13 +1634,14 @@ public class Case { /** * Constructs a Case object for a new Autopsy case. * - * @param caseType The type of case (single-user or multi-user). - * @param caseDir The full path of the case directory. The directory - * will be created if it doesn't already exist; if it - * exists, it is ASSUMED it was created by calling - * createCaseDirectory. - * @param caseDetails Contains details of the case, such as examiner, display name, etc - * + * @param caseType The type of case (single-user or multi-user). + * @param caseDir The full path of the case directory. The directory + * will be created if it doesn't already exist; if it + * exists, it is ASSUMED it was created by calling + * createCaseDirectory. + * @param caseDetails Contains details of the case, such as examiner, + * display name, etc + * */ private Case(CaseType caseType, String caseDir, CaseDetails caseDetails) { metadata = new CaseMetadata(caseType, caseDir, displayNameToUniqueName(caseDetails.getCaseDisplayName()), caseDetails); diff --git a/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java b/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java index 4a702b6df5..e3b36499f0 100644 --- a/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java +++ b/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearchservice; import java.io.Closeable; import org.sleuthkit.autopsy.casemodule.CaseMetadata; import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; /** @@ -50,6 +51,15 @@ public interface KeywordSearchService extends Closeable { */ public void indexArtifact(BlackboardArtifact artifact) throws TskCoreException; + /** + * Add the given Content object to the text index. + * + * @param content The content to index. + * + * @throws TskCoreException + */ + public void index(Content content) throws TskCoreException; + /** * Deletes the keyword search text index for a case. * diff --git a/KeywordSearch/arch.xml b/KeywordSearch/arch.xml new file mode 100644 index 0000000000..65ae82a275 --- /dev/null +++ b/KeywordSearch/arch.xml @@ -0,0 +1,1076 @@ + + +]> + + + + &api-questions; + + + + +

+ XXX no answer for arch-overall +

+
+ + + + + +

+ XXX no answer for arch-quality +

+
+ + + + + +

+ XXX no answer for arch-time +

+
+ + + + + +

+ XXX no answer for arch-usecases +

+
+ + + + + +

+ XXX no answer for arch-what +

+
+ + + + + + + + + + + + +

+ XXX no answer for compat-deprecation +

+
+ + + + + +

+ XXX no answer for compat-i18n +

+
+ + + + + +

+ XXX no answer for compat-standards +

+
+ + + + + +

+ XXX no answer for compat-version +

+
+ + + + + +

+ XXX no answer for dep-jre +

+
+ + + + + +

+ XXX no answer for dep-jrejdk +

+
+ + + + + + + + + + + + +

+ XXX no answer for dep-non-nb +

+
+ + + + + +

+ XXX no answer for dep-platform +

+
+ + + + + +

+ XXX no answer for deploy-dependencies +

+
+ + + + + +

+ XXX no answer for deploy-jar +

+
+ + + + + +

+ XXX no answer for deploy-nbm +

+
+ + + + + +

+ XXX no answer for deploy-packages +

+
+ + + + + +

+ XXX no answer for deploy-shared +

+
+ + + + + +

+ XXX no answer for exec-ant-tasks +

+
+ + + + + +

+ XXX no answer for exec-classloader +

+
+ + + + + +

+ XXX no answer for exec-component +

+
+ + + + + +

+ XXX no answer for exec-introspection +

+
+ + + + + +

+ XXX no answer for exec-privateaccess +

+
+ + + + + +

+ XXX no answer for exec-process +

+
+ + + + + +

+ XXX no answer for exec-property +

+
+ + + + + +

+ XXX no answer for exec-reflection +

+
+ + + + + +

+ XXX no answer for exec-threading +

+
+ + + + + +

+ XXX no answer for format-clipboard +

+
+ + + + + +

+ XXX no answer for format-dnd +

+
+ + + + + +

+ XXX no answer for format-types +

+
+ + + + + +

+ XXX no answer for lookup-lookup +

+
+ + + + + +

+ XXX no answer for lookup-register +

+
+ + + + + +

+ XXX no answer for lookup-remove +

+
+ + + + + +

+ XXX no answer for perf-exit +

+
+ + + + + +

+ XXX no answer for perf-huge_dialogs +

+
+ + + + + +

+ XXX no answer for perf-limit +

+
+ + + + + +

+ XXX no answer for perf-mem +

+
+ + + + + +

+ XXX no answer for perf-menus +

+
+ + + + + +

+ XXX no answer for perf-progress +

+
+ + + + + +

+ XXX no answer for perf-scale +

+
+ + + + + +

+ XXX no answer for perf-spi +

+
+ + + + + +

+ XXX no answer for perf-startup +

+
+ + + + + +

+ XXX no answer for perf-wakeup +

+
+ + + + + +

+ XXX no answer for resources-file +

+
+ + + + + +

+ XXX no answer for resources-layer +

+
+ + + + + +

+ XXX no answer for resources-mask +

+
+ + + + + +

+ XXX no answer for resources-preferences +

+
+ + + + + +

+ XXX no answer for resources-read +

+
+ + + + + +

+ XXX no answer for security-grant +

+
+ + + + + +

+ XXX no answer for security-policy +

+
+ +
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index 94367c5b26..c7f4c07f6a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,12 +31,14 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.keywordsearch.Chunker.Chunk; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.Directory; import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalDirectory; import org.sleuthkit.datamodel.LocalFile; +import org.sleuthkit.datamodel.Report; import org.sleuthkit.datamodel.SlackFile; import org.sleuthkit.datamodel.SleuthkitItemVisitor; import org.sleuthkit.datamodel.SleuthkitVisitableItem; @@ -361,6 +363,31 @@ class Ingester { } return params; } + + /** + * Get the field map for artifacts. + * + * @param report The report to get fields for. + * + * @return The field map for the given report. + */ + @Override + public Map visit(Report report) { + Map params = new HashMap<>(); + params.put(Server.Schema.ID.toString(), Long.toString(report.getId())); + try { + Content dataSource = report.getDataSource(); + if (null == dataSource) { + params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1)); + } else { + params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(dataSource.getId())); + } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Could not get data source id to properly index the report, using default value. Id: " + report.getId(), ex); //NON-NLS + params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1)); + } + return params; + } } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 79174ba134..52e5da17f1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,6 +46,7 @@ import org.sleuthkit.autopsy.progress.ProgressIndicator; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; /** @@ -95,6 +96,35 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { } } + /** + * Add the given Content object to the text index. + * @param content The content to index. + * @throws TskCoreException + */ + @Override + public void index(Content content) throws TskCoreException { + if (content == null) { + return; + } + final Ingester ingester = Ingester.getDefault(); + + try { + ingester.indexText(new TikaTextExtractor(), content, null); + } catch (Ingester.IngesterException ex) { + try { + // Try the StringsTextExtractor if Tika extractions fails. + ingester.indexText(new StringsTextExtractor(), content, null); + } catch (Ingester.IngesterException ex1) { + throw new TskCoreException(ex.getCause().getMessage(), ex1); + } + } + + // TODO: Review whether this is the right thing to do. We typically use + // a combination of autoCommit and the SearchRunner to ensure that data + // is committed but that might not be sufficient for reports (or artifacts). + ingester.commit(); + } + /** * Tries to connect to the keyword search service. * diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TikaTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TikaTextExtractor.java index 961b4a38a0..108962f99a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TikaTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TikaTextExtractor.java @@ -40,7 +40,6 @@ import org.apache.tika.parser.ParsingReader; import org.apache.tika.parser.microsoft.OfficeParserConfig; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ReadContentInputStream; diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index b0af801089..e111f9ae32 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2012-2014 Basis Technology Corp. + * Copyright 2012-2018 Basis Technology Corp. * * Copyright 2012 42six Solutions. * Contact: aebadirad 42six com @@ -50,9 +50,11 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import java.nio.file.Path; +import org.openide.util.Lookup; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; /** * Extract windows registry data using regripper. Runs two versions of @@ -202,7 +204,17 @@ class ExtractRegistry extends Extract { // create a report for the full output if (!regOutputFiles.fullPlugins.isEmpty()) { try { - currentCase.addReport(regOutputFiles.fullPlugins, NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace"), "RegRipper " + regFile.getUniquePath()); //NON-NLS + Report report = currentCase.addReport(regOutputFiles.fullPlugins, + NbBundle.getMessage(this.getClass(), "ExtractRegistry.parentModuleName.noSpace"), + "RegRipper " + regFile.getUniquePath(), regFile); //NON-NLS + + // Index the report content so that it will be available for keyword search. + KeywordSearchService searchService = Lookup.getDefault().lookup(KeywordSearchService.class); + if (null == searchService) { + logger.log(Level.WARNING, "Keyword search service not found. Report will not be indexed"); + } else { + searchService.index(report); + } } catch (TskCoreException e) { this.addErrorMessage("Error adding regripper output as Autopsy report: " + e.getLocalizedMessage()); //NON-NLS } @@ -413,12 +425,14 @@ class ExtractRegistry extends Extract { installtime = Long.valueOf(Tempdate) / 1000; } catch (ParseException e) { logger.log(Level.SEVERE, "RegRipper::Conversion on DateTime -> ", e); //NON-NLS - } break; + } + break; default: break; } } - } try { + } + try { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, parentModuleName, version)); if (installtime != null) { @@ -475,7 +489,8 @@ class ExtractRegistry extends Extract { break; } } - } try { + } + try { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VERSION, parentModuleName, os)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROCESSOR_ARCHITECTURE, parentModuleName, procArch)); @@ -514,7 +529,8 @@ class ExtractRegistry extends Extract { domain = value; } } - } try { + } + try { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, parentModuleName, compName)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, parentModuleName, domain)); @@ -656,7 +672,7 @@ class ExtractRegistry extends Extract { parentModuleName, sid)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH, parentModuleName, homeDir)); - + bbart.addAttributes(bbattributes); // index the artifact for keyword search this.indexArtifact(bbart); From 0e5402b60872fa02fb4c8e43e7677830f6101e16 Mon Sep 17 00:00:00 2001 From: esaunders Date: Mon, 29 Jan 2018 16:28:45 -0500 Subject: [PATCH 2/2] Remove arch.xml --- KeywordSearch/arch.xml | 1076 ---------------------------------------- 1 file changed, 1076 deletions(-) delete mode 100644 KeywordSearch/arch.xml diff --git a/KeywordSearch/arch.xml b/KeywordSearch/arch.xml deleted file mode 100644 index 65ae82a275..0000000000 --- a/KeywordSearch/arch.xml +++ /dev/null @@ -1,1076 +0,0 @@ - - -]> - - - - &api-questions; - - - - -

- XXX no answer for arch-overall -

-
- - - - - -

- XXX no answer for arch-quality -

-
- - - - - -

- XXX no answer for arch-time -

-
- - - - - -

- XXX no answer for arch-usecases -

-
- - - - - -

- XXX no answer for arch-what -

-
- - - - - - - - - - - - -

- XXX no answer for compat-deprecation -

-
- - - - - -

- XXX no answer for compat-i18n -

-
- - - - - -

- XXX no answer for compat-standards -

-
- - - - - -

- XXX no answer for compat-version -

-
- - - - - -

- XXX no answer for dep-jre -

-
- - - - - -

- XXX no answer for dep-jrejdk -

-
- - - - - - - - - - - - -

- XXX no answer for dep-non-nb -

-
- - - - - -

- XXX no answer for dep-platform -

-
- - - - - -

- XXX no answer for deploy-dependencies -

-
- - - - - -

- XXX no answer for deploy-jar -

-
- - - - - -

- XXX no answer for deploy-nbm -

-
- - - - - -

- XXX no answer for deploy-packages -

-
- - - - - -

- XXX no answer for deploy-shared -

-
- - - - - -

- XXX no answer for exec-ant-tasks -

-
- - - - - -

- XXX no answer for exec-classloader -

-
- - - - - -

- XXX no answer for exec-component -

-
- - - - - -

- XXX no answer for exec-introspection -

-
- - - - - -

- XXX no answer for exec-privateaccess -

-
- - - - - -

- XXX no answer for exec-process -

-
- - - - - -

- XXX no answer for exec-property -

-
- - - - - -

- XXX no answer for exec-reflection -

-
- - - - - -

- XXX no answer for exec-threading -

-
- - - - - -

- XXX no answer for format-clipboard -

-
- - - - - -

- XXX no answer for format-dnd -

-
- - - - - -

- XXX no answer for format-types -

-
- - - - - -

- XXX no answer for lookup-lookup -

-
- - - - - -

- XXX no answer for lookup-register -

-
- - - - - -

- XXX no answer for lookup-remove -

-
- - - - - -

- XXX no answer for perf-exit -

-
- - - - - -

- XXX no answer for perf-huge_dialogs -

-
- - - - - -

- XXX no answer for perf-limit -

-
- - - - - -

- XXX no answer for perf-mem -

-
- - - - - -

- XXX no answer for perf-menus -

-
- - - - - -

- XXX no answer for perf-progress -

-
- - - - - -

- XXX no answer for perf-scale -

-
- - - - - -

- XXX no answer for perf-spi -

-
- - - - - -

- XXX no answer for perf-startup -

-
- - - - - -

- XXX no answer for perf-wakeup -

-
- - - - - -

- XXX no answer for resources-file -

-
- - - - - -

- XXX no answer for resources-layer -

-
- - - - - -

- XXX no answer for resources-mask -

-
- - - - - -

- XXX no answer for resources-preferences -

-
- - - - - -

- XXX no answer for resources-read -

-
- - - - - -

- XXX no answer for security-grant -

-
- - - - - -

- XXX no answer for security-policy -

-
- -