mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'sleuthkit:develop' into develop

Browse Source
This commit is contained in:
Seb2lyon 2021-07-09 17:59:48 +02:00 committed by GitHub
commit c5fcf71303
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 1420 additions and 1067 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Core/manifest.mf
View File

@ -2,7 +2,7 @@ Manifest-Version: 1.0
OpenIDE-Module: org.sleuthkit.autopsy.core/10
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/core/Bundle.properties
OpenIDE-Module-Layer: org/sleuthkit/autopsy/core/layer.xml
OpenIDE-Module-Implementation-Version: 35
OpenIDE-Module-Implementation-Version: 36
OpenIDE-Module-Requires: org.openide.windows.WindowManager
AutoUpdate-Show-In-Client: true
AutoUpdate-Essential-Module: true

6
Core/nbproject/project.properties
View File

@ -109,8 +109,8 @@ file.reference.protobuf-java-util-3.7.0.jar=release\\modules\\ext\\protobuf-java
file.reference.Rejistry-1.1-SNAPSHOT.jar=release\\modules\\ext\\Rejistry-1.1-SNAPSHOT.jar
file.reference.sevenzipjbinding-AllPlatforms.jar=release\\modules\\ext\\sevenzipjbinding-AllPlatforms.jar
file.reference.sevenzipjbinding.jar=release\\modules\\ext\\sevenzipjbinding.jar
file.reference.sleuthkit-4.10.2.jar=release/modules/ext/sleuthkit-4.10.2.jar
file.reference.sleuthkit-caseuco-4.10.2.jar=release/modules/ext/sleuthkit-caseuco-4.10.2.jar
file.reference.sleuthkit-4.11.0.jar=release/modules/ext/sleuthkit-4.11.0.jar
file.reference.sleuthkit-caseuco-4.11.0.jar=release/modules/ext/sleuthkit-caseuco-4.11.0.jar
file.reference.slf4j-api-1.7.6.jar=release\\modules\\ext\\slf4j-api-1.7.6.jar
file.reference.slf4j-log4j12-1.7.6.jar=release\\modules\\ext\\slf4j-log4j12-1.7.6.jar
file.reference.SparseBitSet-1.1.jar=release\\modules\\ext\\SparseBitSet-1.1.jar
@ -128,4 +128,4 @@ nbm.homepage=http://www.sleuthkit.org/
nbm.module.author=Brian Carrier
nbm.needs.restart=true
source.reference.curator-recipes-2.8.0.jar=release/modules/ext/curator-recipes-2.8.0-sources.jar
spec.version.base=10.23
spec.version.base=10.24

8
Core/nbproject/project.xml
View File

@ -682,8 +682,8 @@
<binary-origin>release\modules\ext\grpc-alts-1.19.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sleuthkit-caseuco-4.10.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sleuthkit-caseuco-4.10.2.jar</binary-origin>
<runtime-relative-path>ext/sleuthkit-caseuco-4.11.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sleuthkit-caseuco-4.11.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jdom-2.0.5.jar</runtime-relative-path>
@ -802,8 +802,8 @@
<binary-origin>release\modules\ext\sevenzipjbinding-AllPlatforms.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/sleuthkit-4.10.2.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sleuthkit-4.10.2.jar</binary-origin>
<runtime-relative-path>ext/sleuthkit-4.11.0.jar</runtime-relative-path>
<binary-origin>release/modules/ext/sleuthkit-4.11.0.jar</binary-origin>
</class-path-extension>
<class-path-extension>
<runtime-relative-path>ext/jutf7-1.0.0.jar</runtime-relative-path>

1339
Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED
View File

File diff suppressed because it is too large Load Diff

12
Core/src/org/sleuthkit/autopsy/contentviewers/Metadata.java
View File

@ -324,6 +324,7 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
this.node = node;
}
@Messages("MetadataWorker.doInBackground.noDataMsg=No Data")
@Override
protected String doInBackground() throws Exception {
AbstractFile file = node.getLookup().lookup(AbstractFile.class);
@ -397,7 +398,6 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
FsContent fsFile = (FsContent) file;
addHeader(sb, NbBundle.getMessage(this.getClass(), "Metadata.nodeText.text"), true);
startTable(sb);
List<String> istatStrings = Collections.emptyList();
try {
@ -406,6 +406,14 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
istatStrings = Arrays.asList(NbBundle.getMessage(this.getClass(), "Metadata.nodeText.exceptionNotice.text") + ex.getLocalizedMessage());
}
if (istatStrings.isEmpty() || (istatStrings.size() == 1 && StringUtils.isEmpty(istatStrings.get(0)))) {
sb.append(MessageFormat.format("<div class=\"{0}\"><p class=\"{1}\">{2}</p><div>",
ContentViewerHtmlStyles.getIndentedClassName(),
ContentViewerHtmlStyles.getTextClassName(),
Bundle.MetadataWorker_doInBackground_noDataMsg()));
} else {
startTable(sb);
for (String str : istatStrings) {
addMonospacedRow(sb, str);
@ -421,6 +429,7 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
endTable(sb);
}
}
} else {
try {
@ -461,7 +470,6 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
// Add all the data source paths to the "Local Path" value cell.
String[] imagePaths = image.getPaths();
if (imagePaths.length > 0) {
addRowWithMultipleValues(sb,
NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.localPath"),

7
Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java
View File

@ -42,6 +42,7 @@ import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.contentviewers.layout.ContentViewerDefaults;
import org.sleuthkit.autopsy.contentviewers.osaccount.SectionData.RowData;
import org.sleuthkit.autopsy.coreutils.TimeZoneUtils;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.Host;
@ -191,11 +192,7 @@ public class OsAccountDataPanel extends JPanel {
account.getOsAccountType().isPresent() ? account.getOsAccountType().get().getName() : "");
Optional<Long> crTime = account.getCreationTime();
if (crTime.isPresent()) {
data.addData(Bundle.OsAccountDataPanel_basic_creationDate(), DATE_FORMAT.format(new Date(crTime.get() * 1000)));
} else {
data.addData(Bundle.OsAccountDataPanel_basic_creationDate(), "");
}
data.addData(Bundle.OsAccountDataPanel_basic_creationDate(), crTime.isPresent() ? TimeZoneUtils.getFormattedTime(crTime.get()) : "");
return data;
}

4
Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java
View File

@ -207,7 +207,7 @@ public final class OsAccounts implements AutopsyVisitableItem {
Bundle.OsAccounts_accountRealmNameProperty_name(),
Bundle.OsAccounts_accountRealmNameProperty_displayName(),
Bundle.OsAccounts_accountRealmNameProperty_desc(),
""));
realmNames.get(0)));
}
}
}
@ -308,7 +308,7 @@ public final class OsAccounts implements AutopsyVisitableItem {
Optional<Long> creationTimeValue = account.getCreationTime();
String timeDisplayStr
= creationTimeValue.isPresent() ? TimeZoneUtils.getFormattedTime(creationTimeValue.get() * 1000) : "";
= creationTimeValue.isPresent() ? TimeZoneUtils.getFormattedTime(creationTimeValue.get()) : "";
propertiesSet.put(new NodeProperty<>(
Bundle.OsAccounts_createdTimeProperty_name(),

1
Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED
View File

@ -24,6 +24,7 @@ DateFilterPanel.invalidRange.text=Range or Only Last must be selected.
DateFilterPanel.startAfterEnd.text=Start date should be before the end date when both are enabled.
DateFilterPanel.startOrEndNeeded.text=A start or end date must be specified to use the range filter.
DiscoveryDialog.name.text=Discovery
DiscoveryExtractAction.title.extractFiles.text=Extract File
DiscoveryTopComponent.additionalFilters.text=;
DiscoveryTopComponent.cancelButton.text=Cancel Search
DiscoveryTopComponent.domainSearch.text=Type: Domain

5
Core/src/org/sleuthkit/autopsy/discovery/ui/DiscoveryExtractAction.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy
*
* Copyright 2019 Basis Technology Corp.
* Copyright 2019-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -40,8 +40,9 @@ final class DiscoveryExtractAction extends AbstractAction {
*
* @param selectedFiles The files to extract from the current case.
*/
@NbBundle.Messages({"DiscoveryExtractAction.title.extractFiles.text=Extract File"})
DiscoveryExtractAction(Collection<AbstractFile> selectedFiles) {
super(NbBundle.getMessage(DiscoveryExtractAction.class, "DiscoveryExtractAction.title.extractFiles.text"));
super(Bundle.DiscoveryExtractAction_title_extractFiles_text());
files.addAll(selectedFiles);
}

10
Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java
View File

@ -106,14 +106,4 @@ public interface KeywordSearchService extends Closeable {
* @throws KeywordSearchServiceException if unable to delete.
*/
void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException;
/**
* A flag to enable or disable OCR on all future text indexing.
*
* @param state Boolean flag to enable/disable OCR. Set to True to enable
* OCR, or False to disable it.
*/
@Beta
void changeOcrState(boolean state);
}

9
Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2020 Basis Technology Corp.
* Copyright 2011-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -49,7 +49,6 @@ import org.sleuthkit.autopsy.modules.pictureanalyzer.PictureAnalyzerIngestModule
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_USER_CONTENT_SUSPECTED;
import org.sleuthkit.datamodel.BlackboardAttribute;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.Content;
@ -150,7 +149,11 @@ public class EXIFProcessor implements PictureProcessor {
if (!attributes.isEmpty() && !blackboard.artifactExists(file, TSK_METADATA_EXIF, attributes)) {
final BlackboardArtifact exifArtifact = file.newDataArtifact(new BlackboardArtifact.Type(TSK_METADATA_EXIF), attributes);
final BlackboardArtifact exifArtifact = (file.newAnalysisResult(
BlackboardArtifact.Type.TSK_METADATA_EXIF,
Score.SCORE_NONE,
null, null, null,
attributes)).getAnalysisResult();
final BlackboardArtifact userSuspectedArtifact = file.newAnalysisResult(
BlackboardArtifact.Type.TSK_USER_CONTENT_SUSPECTED, Score.SCORE_UNKNOWN, null, null, null,

12
Core/src/org/sleuthkit/autopsy/textextractors/TextExtractor.java
View File

@ -22,6 +22,8 @@ import java.io.Reader;
import java.util.Collections;
import java.util.Map;
import org.openide.util.Lookup;
import org.sleuthkit.autopsy.textextractors.configs.ImageConfig;
import org.sleuthkit.datamodel.AbstractFile;
/**
* Extracts the text out of Content instances and exposes them as a Reader.
@ -73,6 +75,16 @@ public interface TextExtractor {
return Collections.emptyMap();
}
/**
* Returns true if this text extractor, based on the provided settings, will
* perform ocr.
*
* @return True if will perform OCR.
*/
default boolean willUseOCR() {
return false;
}
/**
* System level exception for reader initialization.
*/

119
Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java
View File

@ -18,7 +18,6 @@
*/
package org.sleuthkit.autopsy.textextractors;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharSource;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.File;
@ -71,12 +70,15 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Set;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY;
import org.sleuthkit.autopsy.coreutils.ExecUtil.HybridTerminator;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
/**
* Extracts text from Tika supported content. Protects against Tika parser hangs
@ -86,8 +88,8 @@ final class TikaTextExtractor implements TextExtractor {
//Mimetype groups to aassist extractor implementations in ignoring binary and
//archive files.
private static final List<String> BINARY_MIME_TYPES
= ImmutableList.of(
private static final Set<String> BINARY_MIME_TYPES
= ImmutableSet.of(
//ignore binary blob data, for which string extraction will be used
"application/octet-stream", //NON-NLS
"application/x-msdownload"); //NON-NLS
@ -96,8 +98,8 @@ final class TikaTextExtractor implements TextExtractor {
* generally text extractors should ignore archives and let unpacking
* modules take care of them
*/
private static final List<String> ARCHIVE_MIME_TYPES
= ImmutableList.of(
private static final Set<String> ARCHIVE_MIME_TYPES
= ImmutableSet.of(
//ignore unstructured binary and compressed data, for which string extraction or unzipper works better
"application/x-7z-compressed", //NON-NLS
"application/x-ace-compressed", //NON-NLS
@ -134,22 +136,36 @@ final class TikaTextExtractor implements TextExtractor {
// Used to log to the tika file that is why it uses the java.util.logging.logger class instead of the Autopsy one
private static final java.util.logging.Logger TIKA_LOGGER = java.util.logging.Logger.getLogger("Tika"); //NON-NLS
private static final Logger AUTOPSY_LOGGER = Logger.getLogger(TikaTextExtractor.class.getName());
private static final int LIMITED_OCR_SIZE_MIN = 100 * 1024;
private final ThreadFactory tikaThreadFactory
= new ThreadFactoryBuilder().setNameFormat("tika-reader-%d").build();
private final ExecutorService executorService = Executors.newSingleThreadExecutor(tikaThreadFactory);
private static final String SQLITE_MIMETYPE = "application/x-sqlite3";
private final AutoDetectParser parser = new AutoDetectParser();
private final FileTypeDetector fileTypeDetector;
private final Content content;
private boolean tesseractOCREnabled;
private boolean limitedOCREnabled;
private static final String TESSERACT_DIR_NAME = "Tesseract-OCR"; //NON-NLS
private static final String TESSERACT_EXECUTABLE = "tesseract.exe"; //NON-NLS
private static final File TESSERACT_PATH = locateTesseractExecutable();
private String languagePacks = formatLanguagePacks(PlatformUtil.getOcrLanguagePacks());
private static final String TESSERACT_OUTPUT_FILE_NAME = "tess_output"; //NON-NLS
// documents where OCR is performed
private static final ImmutableSet OCR_DOCUMENTS = ImmutableSet.of(
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
);
private static final String IMAGE_MIME_TYPE_PREFIX = "image/";
private Map<String, String> metadataMap;
private ProcessTerminator processTerminator;
@ -162,18 +178,62 @@ final class TikaTextExtractor implements TextExtractor {
TikaTextExtractor(Content content) {
this.content = content;
FileTypeDetector detector = null;
try {
detector = new FileTypeDetector();
} catch (FileTypeDetector.FileTypeDetectorInitException ex) {
TIKA_LOGGER.log(Level.SEVERE, "Unable to instantiate a file type detector", ex);
}
this.fileTypeDetector = detector;
}
/**
* If Tesseract has been installed and is set to be used through
* configuration, then ocr is enabled. OCR can only currently be run on 64
* bit Windows OS.
* Obtains the mime type of the file using a FileTypeDetector with the
* file's mime type as fallback if the FileTypeDetector is not instantiated.
* If no mime type present, MimeTypes.OCTET_STREAM is returned.
*
* @return Flag indicating if OCR is set to be used.
* @param file The abstract file instance.
*
* @return The mime type or MimeTypes.OCTET_STREAM if the mime type cannot
* be determined.
*/
private boolean ocrEnabled() {
return TESSERACT_PATH != null && tesseractOCREnabled
&& PlatformUtil.isWindowsOS() == true && PlatformUtil.is64BitOS();
private String getMimeType(AbstractFile file) {
String mimeType = MimeTypes.OCTET_STREAM;
if (fileTypeDetector != null) {
mimeType = fileTypeDetector.getMIMEType(file);
} else if (file.getMIMEType() != null) {
mimeType = file.getMIMEType();
}
return mimeType.trim().toLowerCase();
}
@Override
public boolean willUseOCR() {
if (!isOcrSupported() || (!(content instanceof AbstractFile))) {
return false;
}
String mimeType = getMimeType((AbstractFile) content);
// in order to ocr, it needs to either be an image or a document with embedded content
return mimeType.startsWith(IMAGE_MIME_TYPE_PREFIX) || OCR_DOCUMENTS.contains(mimeType);
}
/**
* Whether or not OCR is supported in environment.
*
* @return True if OCR is supported.
*/
private boolean isOcrSupported() {
// If Tesseract has been installed and is set to be used through
// configuration, then ocr is enabled. OCR can only currently be run on 64
// bit Windows OS.
return TESSERACT_PATH != null
&& tesseractOCREnabled
&& PlatformUtil.isWindowsOS()
&& PlatformUtil.is64BitOS()
&& isSupported();
}
/**
@ -195,33 +255,31 @@ final class TikaTextExtractor implements TextExtractor {
// Only abstract files are supported, see isSupported()
final AbstractFile file = ((AbstractFile) content);
// This mime type must be non-null, see isSupported()
final String mimeType = file.getMIMEType();
String mimeType = getMimeType(file);
// Handle images seperately so the OCR task can be cancelled.
// See JIRA-4519 for the need to have cancellation in the UI and ingest.
if (ocrEnabled() && mimeType.toLowerCase().startsWith("image/") && useOcrOnFile(file)) {
if (isOcrSupported() && mimeType.startsWith(IMAGE_MIME_TYPE_PREFIX)) {
InputStream imageOcrStream = performOCR(file);
return new InputStreamReader(imageOcrStream, Charset.forName("UTF-8"));
}
// Set up Tika
final InputStream stream = new ReadContentInputStream(content);
final ParseContext parseContext = new ParseContext();
final ParseContext parseContext = new ParseContext();
// Documents can contain other documents. By adding
// the parser back into the context, Tika will recursively
// parse embedded documents.
parseContext.set(Parser.class, parser);
// Use the more memory efficient Tika SAX parsers for DOCX and
// PPTX files (it already uses SAX for XLSX).
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setUseSAXPptxExtractor(true);
officeParserConfig.setUseSAXDocxExtractor(true);
parseContext.set(OfficeParserConfig.class, officeParserConfig);
if (ocrEnabled() && useOcrOnFile(file)) {
if (isOcrSupported()) {
// Configure OCR for Tika if it chooses to run OCR
// during extraction
TesseractOCRConfig ocrConfig = new TesseractOCRConfig();
@ -347,22 +405,6 @@ final class TikaTextExtractor implements TextExtractor {
}
}
/**
* Method to indicate if OCR should be performed on this image file. Checks
* to see if the limited OCR setting is enabled. If it is it will also check
* that one of the limiting factors is true.
*
* @param file The AbstractFile which OCR might be performed on.
* @param boolean The configuration setting which indicates if limited OCR
* is enabled in Keyword Search.
*
* @return True if limited OCR is not enabled or the image is greater than
* 100KB in size or the image is a derived file.
*/
private boolean useOcrOnFile(AbstractFile file) {
return !limitedOCREnabled || file.getSize() > LIMITED_OCR_SIZE_MIN || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED;
}
/**
* Wraps the creation of a TikaReader into a Future so that it can be
* cancelled.
@ -552,7 +594,6 @@ final class TikaTextExtractor implements TextExtractor {
ImageConfig configInstance = context.lookup(ImageConfig.class);
if (configInstance != null) {
this.tesseractOCREnabled = configInstance.getOCREnabled();
this.limitedOCREnabled = configInstance.getLimitedOCREnabled();
if (Objects.nonNull(configInstance.getOCRLanguages())) {
this.languagePacks = formatLanguagePacks(configInstance.getOCRLanguages());

22
Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java
View File

@ -35,7 +35,6 @@ public class ImageConfig {
private static final int OCR_TIMEOUT_SECONDS = 30 * 60;
private boolean OCREnabled = false;
private boolean limitedOCREnabled = false;
private List<String> ocrLanguages = null;
private final TimedProcessTerminator ocrTimedTerminator = new TimedProcessTerminator(OCR_TIMEOUT_SECONDS);
@ -49,16 +48,6 @@ public class ImageConfig {
this.OCREnabled = enabled;
}
/**
* Enables the limiting OCR to be run on larger images and images which were
* extracted from documents.
*
* @param enabled Flag indicating if OCR is enabled.
*/
public void setLimitedOCREnabled(boolean enabled) {
this.limitedOCREnabled = enabled;
}
/**
* Gets the OCR flag that has been set. By default this flag is turned off.
*
@ -98,15 +87,4 @@ public class ImageConfig {
public ProcessTerminator getOCRTimeoutTerminator() {
return ocrTimedTerminator;
}
/**
* Gets the limited OCR flag to indicate if OCR should be limited to larger
* images and images which were extracted from documents.
*
* @return Flag indicating if limited OCR is enabled. True if OCR should be
* limited, false otherwise..
*/
public boolean getLimitedOCREnabled() {
return limitedOCREnabled;
}
}

4
Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java
View File

@ -1270,7 +1270,7 @@ public class CentralRepoDatamodelTest extends TestCase {
// We expect 10 - the custom type is disabled
// Note: this test will need to be updated based on the current default items defined in the correlation_types table
assertTrue("getDefinedCorrelationTypes returned " + types.size() + " enabled entries - expected 27", types.size() == 27);
assertTrue("getDefinedCorrelationTypes returned " + types.size() + " enabled entries - expected 29", types.size() == 29);
} catch (CentralRepoException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex.getMessage());
@ -1282,7 +1282,7 @@ public class CentralRepoDatamodelTest extends TestCase {
// We expect 10 - the custom type is not supported
// Note: this test will need to be updated based on the current default items defined in the correlation_types table
assertTrue("getDefinedCorrelationTypes returned " + types.size() + " supported entries - expected 27", types.size() == 27);
assertTrue("getDefinedCorrelationTypes returned " + types.size() + " supported entries - expected 29", types.size() == 29);
} catch (CentralRepoException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex.getMessage());

2
Experimental/nbproject/project.xml
View File

@ -144,7 +144,7 @@
<compile-dependency/>
<run-dependency>
<release-version>10</release-version>
<specification-version>10.23</specification-version>
<specification-version>10.24</specification-version>
</run-dependency>
</dependency>
<dependency>

27
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

@ -105,11 +105,11 @@ import org.sleuthkit.autopsy.ingest.IngestModuleError;
import org.sleuthkit.autopsy.ingest.IngestStream;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException;
import org.sleuthkit.autopsy.keywordsearch.Server;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchJobSettings;
/**
* An auto ingest manager is responsible for processing auto ingest jobs defined
@ -2295,15 +2295,6 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
currentJob.setProcessingStage(AutoIngestJob.Stage.UPDATING_SHARED_CONFIG, Date.from(Instant.now()));
new SharedConfiguration().downloadConfiguration();
}
// update the OCR enabled/disabled setting
if (currentJob.getOcrEnabled()) {
sysLogger.log(Level.INFO, "Enabling OCR for job {0}", currentJob.getManifest().getFilePath());
} else {
sysLogger.log(Level.INFO, "Disabling OCR for job {0}", currentJob.getManifest().getFilePath());
}
KeywordSearchService kwsService = Lookup.getDefault().lookup(KeywordSearchService.class);
kwsService.changeOcrState(currentJob.getOcrEnabled());
}
/**
@ -2755,6 +2746,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
}
/**
* Analyzes the data source content returned by the data source
* processor using the configured set of data source level and file
@ -2798,6 +2790,20 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
throw new AnalysisStartupException("Error(s) in ingest job settings");
}
// update the OCR enabled/disabled setting
if (currentJob.getOcrEnabled()) {
sysLogger.log(Level.INFO, "Enabling OCR for job {0}", currentJob.getManifest().getFilePath());
} else {
sysLogger.log(Level.INFO, "Disabling OCR for job {0}", currentJob.getManifest().getFilePath());
}
// find the KeywordSearchJobSettings instance in the templates, and if present, set ocr enabled
ingestJobSettings.getIngestModuleTemplates().stream()
.filter(template -> template.getModuleSettings() instanceof KeywordSearchJobSettings)
.map(template -> (KeywordSearchJobSettings) template.getModuleSettings())
.findFirst()
.ifPresent((keywordJobSettings) -> keywordJobSettings.setOCREnabled(currentJob.getOcrEnabled()));
ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings);
ingestJob = ingestJobStartResult.getJob();
@ -2862,6 +2868,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
}
/**
* Gather metrics to store in auto ingest job nodes. A SleuthkitCase
* instance is used to get the content size.

2
ImageGallery/nbproject/project.xml
View File

@ -127,7 +127,7 @@
<compile-dependency/>
<run-dependency>
<release-version>10</release-version>
<specification-version>10.23</specification-version>
<specification-version>10.24</specification-version>
</run-dependency>
</dependency>
<dependency>

2
KeywordSearch/manifest.mf
View File

@ -1,7 +1,7 @@
Manifest-Version: 1.0
AutoUpdate-Show-In-Client: true
OpenIDE-Module: org.sleuthkit.autopsy.keywordsearch/6
OpenIDE-Module-Implementation-Version: 22
OpenIDE-Module-Implementation-Version: 23
OpenIDE-Module-Install: org/sleuthkit/autopsy/keywordsearch/Installer.class
OpenIDE-Module-Layer: org/sleuthkit/autopsy/keywordsearch/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/keywordsearch/Bundle.properties

2
KeywordSearch/nbproject/project.xml
View File

@ -128,7 +128,7 @@
<compile-dependency/>
<run-dependency>
<release-version>10</release-version>
<specification-version>10.23</specification-version>
<specification-version>10.24</specification-version>
</run-dependency>
</dependency>
<dependency>

5
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
View File

@ -319,5 +319,6 @@ ExtractedContentPanel.pageTotalLabel.text=-
ExtractedContentPanel.pageOfLabel.text=of
ExtractedContentPanel.pageCurLabel.text=-
ExtractedContentPanel.pagesLabel.text=Page:
KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text=Only process images which are over 100KB in size or extracted from a document (Beta)
KeywordSearchJobSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
KeywordSearchJobSettingsPanel.limitedOcrCheckbox.text=<html>Only process PDFs, MS Office docs and images which are over 100KB in size or extracted from another file (Beta)</html>
KeywordSearchJobSettingsPanel.ocrOnlyCheckbox.text=Only index text extracted using OCR

5
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED
View File

@ -402,8 +402,9 @@ ExtractedContentPanel.pageTotalLabel.text=-
ExtractedContentPanel.pageOfLabel.text=of
ExtractedContentPanel.pageCurLabel.text=-
ExtractedContentPanel.pagesLabel.text=Page:
KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text=Only process images which are over 100KB in size or extracted from a document (Beta)
KeywordSearchJobSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
KeywordSearchJobSettingsPanel.limitedOcrCheckbox.text=<html>Only process images which are over 100KB in size or extracted from a document (Beta)</html>
KeywordSearchJobSettingsPanel.ocrOnlyCheckbox.text=Only index text extracted using OCR
TextZoomPanel.zoomInButton.text=
TextZoomPanel.zoomOutButton.text=
TextZoomPanel.zoomResetButton.text=Reset

1
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties
View File

@ -388,3 +388,4 @@ SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata\u306b\u306f\u
SolrSearchService.exceptionMessage.noIndexMetadata=\u30b1\u30fc\u30b9\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u30fc\u304b\u3089\u6b21\u306eIndexMetaData\u3092\u4f5c\u6210\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\: {0}
SolrSearchService.exceptionMessage.unableToDeleteCollection=\u30b3\u30ec\u30af\u30b7\u30e7\u30f3{0}\u3092\u524a\u9664\u3067\u304d\u307e\u305b\u3093
TextZoomPanel.zoomResetButton.text=\u30ea\u30bb\u30c3\u30c8
KeywordSearchJobSettingsPanel.ocrCheckBox.text=OCR\u6587\u5b57\u8a8d\u8b58\u3092\u6709\u52b9\u306b\u3059\u308b

83
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form
View File

@ -23,25 +23,25 @@
<Group type="102" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="ingestWarningLabel" alignment="0" max="32767" attributes="0"/>
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<Component id="settingsLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="settingsSeparator" min="-2" pref="326" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<Component id="informationLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="informationSeparator" min="-2" pref="309" max="-2" attributes="0"/>
</Group>
<Group type="102" attributes="0">
<EmptySpace min="-2" pref="16" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="skipNSRLCheckBox" min="-2" max="-2" attributes="0"/>
<Component id="showSnippetsCB" alignment="0" min="-2" max="-2" attributes="0"/>
<Component id="ocrCheckBox" alignment="0" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="32767" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="ingestWarningLabel" alignment="0" max="32767" attributes="0"/>
<Group type="102" alignment="0" attributes="0">
<Component id="informationLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="informationSeparator" min="-2" pref="309" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="16" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<Component id="filesIndexedLabel" linkSize="1" min="-2" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
@ -49,7 +49,12 @@
</Group>
<Component id="frequencyLabel" alignment="0" min="-2" max="-2" attributes="0"/>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="16" max="-2" attributes="0"/>
<Component id="chunksLabel" linkSize="1" min="-2" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
<Component id="chunksValLabel" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="16" pref="16" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="timeRadioButton2" min="-2" max="-2" attributes="0"/>
<Component id="timeRadioButton1" min="-2" max="-2" attributes="0"/>
@ -58,22 +63,18 @@
<Component id="timeRadioButton5" alignment="0" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</Group>
</Group>
</Group>
<EmptySpace max="32767" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<Component id="chunksLabel" linkSize="1" min="-2" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
<Component id="chunksValLabel" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" attributes="0">
<EmptySpace min="-2" pref="16" max="-2" attributes="0"/>
<Component id="limitedOcrCheckbox" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</Group>
</Group>
<EmptySpace min="0" pref="0" max="32767" attributes="0"/>
</Group>
</Group>
<Component id="settingsLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="settingsSeparator" min="-2" pref="326" max="-2" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
</Group>
</Group>
</Group>
</Group>
</DimensionLayout>
@ -90,10 +91,6 @@
<EmptySpace max="-2" attributes="0"/>
<Component id="showSnippetsCB" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="ocrCheckBox" min="-2" max="-2" attributes="0"/>
<EmptySpace min="0" pref="0" max="-2" attributes="0"/>
<Component id="limitedOcrCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="frequencyLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="timeRadioButton1" min="-2" max="-2" attributes="0"/>
@ -279,25 +276,5 @@
</Property>
</Properties>
</Component>
<Component class="javax.swing.JCheckBox" name="ocrCheckBox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="ocrCheckBoxActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JCheckBox" name="limitedOcrCheckbox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="limitedOcrCheckboxActionPerformed"/>
</Events>
</Component>
</SubComponents>
</Form>

80
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java
View File

@ -50,13 +50,9 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
private void activateWidgets() {
skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown());
showSnippetsCB.setSelected(KeywordSearchSettings.getShowSnippets());
ocrCheckBox.setSelected(KeywordSearchSettings.getOcrOption());
limitedOcrCheckbox.setSelected(KeywordSearchSettings.getLimitedOcrOption());
boolean ingestRunning = IngestManager.getInstance().isIngestRunning();
ingestWarningLabel.setVisible(ingestRunning);
skipNSRLCheckBox.setEnabled(!ingestRunning);
ocrCheckBox.setEnabled(!ingestRunning);
limitedOcrCheckbox.setEnabled(ocrCheckBox.isSelected() && !ingestRunning);
setTimeSettingEnabled(!ingestRunning);
final UpdateFrequency curFreq = KeywordSearchSettings.getUpdateFrequency();
@ -111,8 +107,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
showSnippetsCB = new javax.swing.JCheckBox();
timeRadioButton5 = new javax.swing.JRadioButton();
ingestWarningLabel = new javax.swing.JLabel();
ocrCheckBox = new javax.swing.JCheckBox();
limitedOcrCheckbox = new javax.swing.JCheckBox();
skipNSRLCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text")); // NOI18N
skipNSRLCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText")); // NOI18N
@ -186,20 +180,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
ingestWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/hashdatabase/warning16.png"))); // NOI18N
ingestWarningLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.ingestWarningLabel.text")); // NOI18N
ocrCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text")); // NOI18N
ocrCheckBox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
ocrCheckBoxActionPerformed(evt);
}
});
limitedOcrCheckbox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text")); // NOI18N
limitedOcrCheckbox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
limitedOcrCheckboxActionPerformed(evt);
}
});
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
@ -207,13 +187,15 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(ingestWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addGroup(layout.createSequentialGroup()
.addGap(16, 16, 16)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(skipNSRLCheckBox)
.addComponent(showSnippetsCB))
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addComponent(settingsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(settingsSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 326, javax.swing.GroupLayout.PREFERRED_SIZE))
.addComponent(ingestWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addGroup(layout.createSequentialGroup()
.addComponent(informationLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
@ -221,14 +203,15 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addGroup(layout.createSequentialGroup()
.addGap(16, 16, 16)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(skipNSRLCheckBox)
.addComponent(showSnippetsCB)
.addComponent(ocrCheckBox)
.addGroup(layout.createSequentialGroup()
.addComponent(filesIndexedLabel)
.addGap(18, 18, 18)
.addComponent(filesIndexedValue))
.addComponent(frequencyLabel)
.addGroup(layout.createSequentialGroup()
.addComponent(chunksLabel)
.addGap(18, 18, 18)
.addComponent(chunksValLabel))
.addGroup(layout.createSequentialGroup()
.addGap(16, 16, 16)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@ -236,16 +219,13 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addComponent(timeRadioButton1)
.addComponent(timeRadioButton3)
.addComponent(timeRadioButton4)
.addComponent(timeRadioButton5)))
.addComponent(timeRadioButton5))))))
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGroup(layout.createSequentialGroup()
.addComponent(chunksLabel)
.addGap(18, 18, 18)
.addComponent(chunksValLabel))
.addGroup(layout.createSequentialGroup()
.addGap(16, 16, 16)
.addComponent(limitedOcrCheckbox)))))
.addGap(0, 0, Short.MAX_VALUE)))
.addContainerGap())
.addComponent(settingsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(settingsSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 326, javax.swing.GroupLayout.PREFERRED_SIZE)
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
);
layout.linkSize(javax.swing.SwingConstants.HORIZONTAL, new java.awt.Component[] {chunksLabel, filesIndexedLabel});
@ -262,10 +242,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(showSnippetsCB)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(ocrCheckBox)
.addGap(0, 0, 0)
.addComponent(limitedOcrCheckbox)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(frequencyLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(timeRadioButton1)
@ -323,15 +299,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_timeRadioButton4ActionPerformed
private void ocrCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ocrCheckBoxActionPerformed
limitedOcrCheckbox.setEnabled(ocrCheckBox.isSelected());
firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_ocrCheckBoxActionPerformed
private void limitedOcrCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_limitedOcrCheckboxActionPerformed
firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_limitedOcrCheckboxActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JLabel chunksLabel;
private javax.swing.JLabel chunksValLabel;
@ -341,8 +308,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
private javax.swing.JLabel informationLabel;
private javax.swing.JSeparator informationSeparator;
private javax.swing.JLabel ingestWarningLabel;
private javax.swing.JCheckBox limitedOcrCheckbox;
private javax.swing.JCheckBox ocrCheckBox;
private javax.swing.JLabel settingsLabel;
private javax.swing.JSeparator settingsSeparator;
private javax.swing.JCheckBox showSnippetsCB;
@ -360,8 +325,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
KeywordSearchSettings.setSkipKnown(skipNSRLCheckBox.isSelected());
KeywordSearchSettings.setUpdateFrequency(getSelectedTimeValue());
KeywordSearchSettings.setShowSnippets(showSnippetsCB.isSelected());
KeywordSearchSettings.setOcrOption(ocrCheckBox.isSelected());
KeywordSearchSettings.setLimitedOcrOption(limitedOcrCheckbox.isSelected());
}
@Override
@ -412,15 +375,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
logger.log(Level.WARNING, "Could not get number of indexed files/chunks"); //NON-NLS
}
if (!PlatformUtil.isWindowsOS() || !PlatformUtil.is64BitOS()) {
ocrCheckBox.setText(Bundle.KeywordSearchGlobalSearchSettingsPanel_customizeComponents_windowsOCR());
ocrCheckBox.setSelected(false);
ocrCheckBox.setEnabled(false);
limitedOcrCheckbox.setSelected(false);
limitedOcrCheckbox.setEnabled(false);
limitedOcrCheckbox.setText(Bundle.KeywordSearchGlobalSearchSettingsPanel_customizeComponents_windowsLimitedOCR());
}
KeywordSearch.addNumIndexedFilesChangeListener(
new PropertyChangeListener() {
@Override

140
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2019 Basis Technology Corp.
* Copyright 2011-2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.keywordsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.CharSource;
import java.io.IOException;
import java.io.Reader;
@ -32,6 +33,7 @@ import java.util.HashMap;
import java.util.List;
import static java.util.Locale.US;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.stream.Collectors;
@ -43,7 +45,6 @@ import org.openide.util.lookup.Lookups;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.ExecUtil.ProcessTerminator;
import org.sleuthkit.autopsy.coreutils.ExecUtil.TimedProcessTerminator;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
@ -87,6 +88,8 @@ import org.sleuthkit.datamodel.TskData.FileKnown;
})
public final class KeywordSearchIngestModule implements FileIngestModule {
private static final int LIMITED_OCR_SIZE_MIN = 100 * 1024;
/**
* generally text extractors should ignore archives and let unpacking
* modules take care of them
@ -145,6 +148,18 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
.put("pdf:PDFVersion", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VERSION)
.build();
private static final String IMAGE_MIME_TYPE_PREFIX = "image/";
// documents where OCR is performed
private static final ImmutableSet<String> OCR_DOCUMENTS = ImmutableSet.of(
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
);
/**
* Options for this extractor
@ -350,12 +365,31 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
return ProcessResult.OK;
}
// if ocr only is enabled and not an ocr file, return
Optional<TextExtractor> extractorOpt = getExtractor(abstractFile);
String mimeType = fileTypeDetector.getMIMEType(abstractFile).trim().toLowerCase();
if (settings.isOCREnabled()) {
// if ocr only and the extractor is not present or will not perform ocr on this file, continue
if (settings.isOCROnly() && (!extractorOpt.isPresent() || !extractorOpt.get().willUseOCR())) {
return ProcessResult.OK;
}
// if limited ocr is enabled, the extractor will use ocr, and
// the file would not be subject to limited ocr reading, continue
if (settings.isLimitedOCREnabled() && extractorOpt.isPresent()
&& extractorOpt.get().willUseOCR() && !isLimitedOCRFile(abstractFile, mimeType)) {
return ProcessResult.OK;
}
}
if (KeywordSearchSettings.getSkipKnown() && abstractFile.getKnown().equals(FileKnown.KNOWN)) {
//index meta-data only
if (context.fileIngestIsCancelled()) {
return ProcessResult.OK;
}
indexer.indexFile(abstractFile, false);
indexer.indexFile(extractorOpt, abstractFile, mimeType, false);
return ProcessResult.OK;
}
@ -363,7 +397,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return ProcessResult.OK;
}
indexer.indexFile(abstractFile, true);
indexer.indexFile(extractorOpt, abstractFile, mimeType, true);
// Start searching if it hasn't started already
if (!startedSearching) {
@ -427,6 +461,29 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
initialized = false;
}
/**
* Returns true if file should have OCR performed on it when limited OCR
* setting is specified.
*
* @param aFile The abstract file.
* @param mimeType The file mime type.
*
* @return True if file should have text extracted when limited OCR setting
* is on.
*/
private boolean isLimitedOCRFile(AbstractFile aFile, String mimeType) {
if (OCR_DOCUMENTS.contains(mimeType)) {
return true;
}
if (mimeType.startsWith(IMAGE_MIME_TYPE_PREFIX)) {
return aFile.getSize() > LIMITED_OCR_SIZE_MIN
|| aFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED;
}
return false;
}
/**
* Posts inbox message with summary of text_ingested files
*/
@ -489,6 +546,18 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
}
private Optional<TextExtractor> getExtractor(AbstractFile abstractFile) {
ImageConfig imageConfig = new ImageConfig();
imageConfig.setOCREnabled(settings.isOCREnabled());
ProcessTerminator terminator = () -> context.fileIngestIsCancelled();
Lookup extractionContext = Lookups.fixed(imageConfig, terminator);
try {
return Optional.ofNullable(TextExtractorFactory.getExtractor(abstractFile, extractionContext));
} catch (TextExtractorFactory.NoTextExtractorFound ex) {
return Optional.empty();
}
}
/**
* File indexer, processes and indexes known/allocated files,
* unknown/unallocated files and directories accordingly
@ -502,25 +571,26 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
* streaming) from the file Divide the file into chunks and index the
* chunks
*
* @param aFile file to extract strings from, divide into chunks and
* index
* @param extractedMetadata Map that will be populated with the file's metadata.
* @param extractorOptional The textExtractor to use with this file or
* empty.
* @param aFile file to extract strings from, divide into
* chunks and index
* @param extractedMetadata Map that will be populated with the file's
* metadata.
*
* @return true if the file was text_ingested, false otherwise
*
* @throws IngesterException exception thrown if indexing failed
*/
private boolean extractTextAndIndex(AbstractFile aFile, Map<String, String> extractedMetadata) throws IngesterException {
ImageConfig imageConfig = new ImageConfig();
imageConfig.setOCREnabled(KeywordSearchSettings.getOcrOption());
imageConfig.setLimitedOCREnabled(KeywordSearchSettings.getLimitedOcrOption());
ProcessTerminator terminator = () -> context.fileIngestIsCancelled();
Lookup extractionContext = Lookups.fixed(imageConfig, terminator);
private boolean extractTextAndIndex(Optional<TextExtractor> extractorOptional, AbstractFile aFile,
Map<String, String> extractedMetadata) throws IngesterException {
try {
TextExtractor extractor = TextExtractorFactory.getExtractor(aFile, extractionContext);
if (!extractorOptional.isPresent()) {
return false;
}
TextExtractor extractor = extractorOptional.get();
Reader fileText = extractor.getReader();
Reader finalReader;
try {
Map<String, String> metadata = extractor.getMetadata();
@ -547,8 +617,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
//divide into chunks and index
return Ingester.getDefault().indexText(finalReader, aFile.getId(), aFile.getName(), aFile, context);
} catch (TextExtractorFactory.NoTextExtractorFound | TextExtractor.InitReaderException ex) {
//No text extractor found... run the default instead
} catch (TextExtractor.InitReaderException ex) {
// Text extractor could not be initialized. No text will be extracted.
return false;
}
}
@ -577,18 +647,17 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
return;
}
if (!bbartifacts.isEmpty()) {
try{
try {
Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard().postArtifacts(bbartifacts, moduleName);
} catch (NoCurrentCaseException | Blackboard.BlackboardException ex) {
// Log error and return to continue processing
logger.log(Level.WARNING, String.format("Unable to post blackboard artifacts for file $s.", aFile.getParentPath() + aFile.getName()) , ex); //NON-NLS
logger.log(Level.WARNING, String.format("Unable to post blackboard artifacts for file $s.", aFile.getParentPath() + aFile.getName()), ex); //NON-NLS
return;
}
}
}
}
private BlackboardAttribute checkAttribute(String key, String value) {
String moduleName = KeywordSearchIngestModule.class.getName();
if (!value.isEmpty() && value.charAt(0) != ' ') {
@ -596,9 +665,9 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
SimpleDateFormat metadataDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", US);
Long metadataDateTime = Long.valueOf(0);
try {
String metadataDate = value.replaceAll("T"," ").replaceAll("Z", "");
String metadataDate = value.replaceAll("T", " ").replaceAll("Z", "");
Date usedDate = metadataDateFormat.parse(metadataDate);
metadataDateTime = usedDate.getTime()/1000;
metadataDateTime = usedDate.getTime() / 1000;
return new BlackboardAttribute(METADATA_TYPES_MAP.get(key), moduleName, metadataDateTime);
} catch (ParseException ex) {
// catching error and displaying date that could not be parsed then will continue on.
@ -614,7 +683,6 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
/**
* Pretty print the text extractor metadata.
*
@ -668,21 +736,24 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
/**
* Adds the file to the index. Detects file type, calls extractors, etc.
*
* @param aFile File to analyze
* @param extractor The textExtractor to use with this file or empty
* if no extractor found.
* @param aFile File to analyze.
* @param mimeType The file mime type.
* @param indexContent False if only metadata should be text_ingested.
* True if content and metadata should be index.
*/
private void indexFile(AbstractFile aFile, boolean indexContent) {
private void indexFile(Optional<TextExtractor> extractor, AbstractFile aFile, String mimeType, boolean indexContent) {
//logger.log(Level.INFO, "Processing AbstractFile: " + abstractFile.getName());
TskData.TSK_DB_FILES_TYPE_ENUM aType = aFile.getType();
/**
* Extract unicode strings from unallocated and unused blocks and
* carved text files. The reason for performing string extraction
* on these is because they all may contain multiple encodings which
* can cause text to be missed by the more specialized text extractors
* used below.
* carved text files. The reason for performing string extraction on
* these is because they all may contain multiple encodings which
* can cause text to be missed by the more specialized text
* extractors used below.
*/
if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
|| aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))
@ -714,11 +785,10 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return;
}
String fileType = fileTypeDetector.getMIMEType(aFile);
// we skip archive formats that are opened by the archive module.
// @@@ We could have a check here to see if the archive module was enabled though...
if (ARCHIVE_MIME_TYPES.contains(fileType)) {
if (ARCHIVE_MIME_TYPES.contains(mimeType)) {
try {
if (context.fileIngestIsCancelled()) {
return;
@ -741,11 +811,11 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return;
}
if (fileType.equals(MimeTypes.OCTET_STREAM)) {
if (MimeTypes.OCTET_STREAM.equals(mimeType)) {
extractStringsAndIndex(aFile);
return;
}
if (!extractTextAndIndex(aFile, extractedMetadata)) {
if (!extractTextAndIndex(extractor, aFile, extractedMetadata)) {
// Text extractor not found for file. Extract string only.
putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
} else {
@ -783,8 +853,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
/**
* Adds the text file to the index given an encoding.
* Returns true if indexing was successful and false otherwise.
* Adds the text file to the index given an encoding. Returns true if
* indexing was successful and false otherwise.
*
* @param aFile Text file to analyze
*/

100
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java
View File

@ -26,19 +26,35 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
/**
* Ingest job settings for the keywords search module.
*/
final class KeywordSearchJobSettings implements IngestModuleIngestJobSettings {
public final class KeywordSearchJobSettings implements IngestModuleIngestJobSettings {
private static final long serialVersionUID = 1L;
private HashSet<String> namesOfEnabledKeywordLists;
private HashSet<String> namesOfDisabledKeywordLists; // Added in version 1.1
/**
* These are nullable so that if no serialized setting, the setting can
* defer to legacy KeywordSearchSettings.
*/
private Boolean ocrEnabled;
private Boolean limitedOCREnabled;
private boolean ocrOnly;
/**
* Constructs ingest job settings for the keywords search module.
*
* @param namesOfEnabledKeywordLists A list of enabled keywords lists.
*/
KeywordSearchJobSettings(List<String> namesOfEnabledKeywordLists) {
this(namesOfEnabledKeywordLists, new ArrayList<String>());
this.namesOfEnabledKeywordLists = new HashSet<>(namesOfEnabledKeywordLists);
this.namesOfDisabledKeywordLists = new HashSet<>();
// explicitly set to default value
this.ocrEnabled = null;
this.limitedOCREnabled = null;
this.ocrOnly = false;
}
/**
@ -46,10 +62,88 @@ final class KeywordSearchJobSettings implements IngestModuleIngestJobSettings {
*
* @param namesOfEnabledKeywordLists A list of enabled keywords lists.
* @param namesOfDisabledKeywordLists A list of disabled keywords lists.
* @param ocrEnabled Whether or not OCR is enabled for
* keyword search.
* @param limitedOCREnabled If true, OCR is to be performed only
* on images larger than 100KB.
* @param ocrOnly True if keyword search ingest should
* be solely limited to OCR.
*/
KeywordSearchJobSettings(List<String> namesOfEnabledKeywordLists, List<String> namesOfDisabledKeywordLists) {
KeywordSearchJobSettings(List<String> namesOfEnabledKeywordLists, List<String> namesOfDisabledKeywordLists, boolean ocrEnabled, boolean limitedOCREnabled, boolean ocrOnly) {
this.namesOfEnabledKeywordLists = new HashSet<>(namesOfEnabledKeywordLists);
this.namesOfDisabledKeywordLists = new HashSet<>(namesOfDisabledKeywordLists);
this.ocrEnabled = ocrEnabled;
this.limitedOCREnabled = limitedOCREnabled;
this.ocrOnly = ocrOnly;
}
/**
* Whether or not OCR is enabled for keyword search.
*
* @return Whether or not OCR is enabled for keyword search.
*/
@SuppressWarnings("deprecation")
public boolean isOCREnabled() {
if (ocrEnabled == null) {
ocrEnabled = KeywordSearchSettings.getOcrOption();
}
return ocrEnabled;
}
/**
* Sets whether or not OCR is enabled for keyword search.
*
* @param ocrEnabled Whether or not OCR is enabled for keyword search.
*/
public void setOCREnabled(boolean ocrEnabled) {
this.ocrEnabled = ocrEnabled;
}
/**
* Returns true if OCR is to be performed only on images larger than 100KB.
* May defer to KeywordSearchSettings if no setting serialized.
*
* @return If true, OCR is to be performed only on images larger than 100KB.
*/
@SuppressWarnings("deprecation")
boolean isLimitedOCREnabled() {
if (limitedOCREnabled == null) {
limitedOCREnabled = KeywordSearchSettings.getLimitedOcrOption();
}
return limitedOCREnabled;
}
/**
* Sets whether or not OCR should be performed only on images larger than
* 100KB.
*
* @param limitedOCREnabled Whether or not OCR should be performed only on
* images larger than 100KB.
*/
void setLimitedOCREnabled(boolean limitedOCREnabled) {
this.limitedOCREnabled = limitedOCREnabled;
}
/**
* Returns true if keyword search ingest should be solely limited to OCR.
*
* @return True if keyword search ingest should be solely limited to OCR.
*/
boolean isOCROnly() {
return ocrOnly;
}
/**
* Sets whether or not keyword search ingest should be solely limited to
* OCR.
*
* @param ocrOnly Whether or not keyword search ingest should be solely
* limited to OCR.
*/
void setOCROnly(boolean ocrOnly) {
this.ocrOnly = ocrOnly;
}
/**

88
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.form
View File

@ -20,30 +20,34 @@
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Group type="103" groupAlignment="1" attributes="0">
<Group type="102" attributes="0">
<Component id="listsScrollPane" pref="0" max="32767" attributes="1"/>
<EmptySpace max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="10" pref="10" max="-2" attributes="0"/>
<Component id="languagesValLabel" min="-2" pref="274" max="-2" attributes="0"/>
<EmptySpace min="10" max="32767" attributes="0"/>
</Group>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="languagesLabel" alignment="1" max="32767" attributes="0"/>
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="listsScrollPane" pref="316" max="32767" attributes="1"/>
<Component id="titleLabel" min="-2" max="-2" attributes="0"/>
<Group type="102" alignment="0" attributes="0">
<Component id="encodingsLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="keywordSearchEncodings" min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="10" pref="10" max="-2" attributes="0"/>
<Component id="languagesValLabel" min="-2" pref="274" max="-2" attributes="0"/>
</Group>
<EmptySpace min="0" pref="0" max="32767" attributes="0"/>
<Component id="ocrCheckBox" alignment="0" min="-2" max="-2" attributes="0"/>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="21" pref="21" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="ocrOnlyCheckbox" min="-2" max="-2" attributes="0"/>
<Component id="limitedOcrCheckbox" min="-2" pref="288" max="-2" attributes="0"/>
</Group>
</Group>
</Group>
<EmptySpace max="-2" attributes="0"/>
</Group>
</Group>
</Group>
@ -54,18 +58,24 @@
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="7" max="-2" attributes="0"/>
<Component id="titleLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="listsScrollPane" max="32767" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="listsScrollPane" pref="41" max="32767" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="languagesLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="languagesLabel" min="-2" pref="13" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="languagesValLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<EmptySpace type="unrelated" min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="encodingsLabel" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="keywordSearchEncodings" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="-2" attributes="0"/>
<EmptySpace type="unrelated" min="-2" max="-2" attributes="0"/>
<Component id="ocrCheckBox" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="ocrOnlyCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="limitedOcrCheckbox" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
@ -96,8 +106,15 @@
<Property name="model" type="javax.swing.table.TableModel" editor="org.netbeans.modules.form.editors2.TableModelEditor">
<Table columnCount="0" rowCount="0"/>
</Property>
<Property name="showHorizontalLines" type="boolean" value="false"/>
<Property name="showVerticalLines" type="boolean" value="false"/>
<Property name="maximumSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[32767, 32767]"/>
</Property>
<Property name="minimumSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[20, 200]"/>
</Property>
<Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="null"/>
</Property>
</Properties>
<AuxValues>
<AuxValue name="JavaCodeGenerator_AddingCodePost" type="java.lang.String" value="listsTable.setDefaultRenderer(String.class, new SimpleTableCellRenderer());"/>
@ -150,5 +167,36 @@
</Property>
</Properties>
</Component>
<Component class="javax.swing.JCheckBox" name="ocrCheckBox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchJobSettingsPanel.ocrCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="ocrCheckBoxActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JCheckBox" name="limitedOcrCheckbox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchJobSettingsPanel.limitedOcrCheckbox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="verticalTextPosition" type="int" value="1"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="limitedOcrCheckboxActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JCheckBox" name="ocrOnlyCheckbox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchJobSettingsPanel.ocrOnlyCheckbox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="ocrOnlyCheckboxActionPerformed"/>
</Events>
</Component>
</SubComponents>
</Form>

114
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.java
View File

@ -18,19 +18,17 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.awt.Component;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.swing.JLabel;
import javax.swing.JTable;
import javax.swing.ListSelectionModel;
import javax.swing.table.AbstractTableModel;
import javax.swing.table.DefaultTableCellRenderer;
import javax.swing.table.TableColumn;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
import org.sleuthkit.autopsy.guiutils.SimpleTableCellRenderer;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
@ -42,16 +40,16 @@ import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.StringsExtr
*/
@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives
public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSettingsPanel implements PropertyChangeListener {
private final KeywordListsTableModel tableModel = new KeywordListsTableModel();
private final List<String> keywordListNames = new ArrayList<>();
private final Map<String, Boolean> keywordListStates = new HashMap<>();
private final XmlKeywordSearchList keywordListsManager = XmlKeywordSearchList.getCurrent();
KeywordSearchJobSettingsPanel(KeywordSearchJobSettings initialSettings) {
initializeKeywordListSettings(initialSettings);
initComponents();
customizeComponents();
initializeKeywordListSettings(initialSettings);
}
private void initializeKeywordListSettings(KeywordSearchJobSettings settings) {
@ -63,6 +61,23 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
keywordListNames.add(listName);
keywordListStates.put(listName, settings.keywordListIsEnabled(listName));
}
ocrCheckBox.setSelected(settings.isOCREnabled());
limitedOcrCheckbox.setSelected(settings.isLimitedOCREnabled());
ocrOnlyCheckbox.setSelected(settings.isOCROnly());
handleOcrEnabled(settings.isOCREnabled());
}
/**
* Handles setting enabled state of checkbox.
* @param ocrEnabled Whether or not the ocr setting is enabled.
*/
private void handleOcrEnabled(boolean ocrEnabled) {
boolean platformSupported = PlatformUtil.isWindowsOS() && PlatformUtil.is64BitOS();
ocrCheckBox.setEnabled(platformSupported);
limitedOcrCheckbox.setEnabled(platformSupported && ocrEnabled);
ocrOnlyCheckbox.setEnabled(platformSupported && ocrEnabled);
}
private void customizeComponents() {
@ -71,6 +86,10 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
displayEncodings();
keywordListsManager.addPropertyChangeListener(this);
languagesLabel.setText("<html>" + org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.languagesLabel.text") + "</html>"); // NOI18N NON-NLS
// the gui builder does not explicitly set these to false.
listsTable.setShowHorizontalLines(false);
listsTable.setShowVerticalLines(false);
}
private void customizeKeywordListsTable() {
@ -174,7 +193,8 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
disabledListNames.add(listName);
}
}
return new KeywordSearchJobSettings(enabledListNames, disabledListNames);
return new KeywordSearchJobSettings(enabledListNames, disabledListNames,
this.ocrCheckBox.isSelected(), this.limitedOcrCheckbox.isSelected(), this.ocrOnlyCheckbox.isSelected());
}
void reset(KeywordSearchJobSettings newSettings) {
@ -241,6 +261,9 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
languagesValLabel = new javax.swing.JLabel();
encodingsLabel = new javax.swing.JLabel();
keywordSearchEncodings = new javax.swing.JLabel();
ocrCheckBox = new javax.swing.JCheckBox();
limitedOcrCheckbox = new javax.swing.JCheckBox();
ocrOnlyCheckbox = new javax.swing.JCheckBox();
setPreferredSize(new java.awt.Dimension(300, 170));
@ -256,8 +279,9 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
}
));
listsTable.setShowHorizontalLines(false);
listsTable.setShowVerticalLines(false);
listsTable.setMaximumSize(new java.awt.Dimension(32767, 32767));
listsTable.setMinimumSize(new java.awt.Dimension(20, 200));
listsTable.setPreferredSize(null);
listsScrollPane.setViewportView(listsTable);
listsTable.setDefaultRenderer(String.class, new SimpleTableCellRenderer());
@ -275,29 +299,54 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
keywordSearchEncodings.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.keywordSearchEncodings.text")); // NOI18N
ocrCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.ocrCheckBox.text")); // NOI18N
ocrCheckBox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
ocrCheckBoxActionPerformed(evt);
}
});
limitedOcrCheckbox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.limitedOcrCheckbox.text")); // NOI18N
limitedOcrCheckbox.setVerticalTextPosition(javax.swing.SwingConstants.TOP);
limitedOcrCheckbox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
limitedOcrCheckboxActionPerformed(evt);
}
});
ocrOnlyCheckbox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.ocrOnlyCheckbox.text")); // NOI18N
ocrOnlyCheckbox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
ocrOnlyCheckboxActionPerformed(evt);
}
});
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addComponent(listsScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)
.addContainerGap())
.addGroup(layout.createSequentialGroup()
.addGap(10, 10, 10)
.addComponent(languagesValLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 274, javax.swing.GroupLayout.PREFERRED_SIZE)
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addComponent(languagesLabel, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(listsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 316, Short.MAX_VALUE)
.addComponent(titleLabel)
.addGroup(layout.createSequentialGroup()
.addComponent(encodingsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(keywordSearchEncodings)))
.addGap(0, 0, Short.MAX_VALUE))))
.addComponent(keywordSearchEncodings))
.addGroup(layout.createSequentialGroup()
.addGap(10, 10, 10)
.addComponent(languagesValLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 274, javax.swing.GroupLayout.PREFERRED_SIZE))
.addComponent(ocrCheckBox)
.addGroup(layout.createSequentialGroup()
.addGap(21, 21, 21)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(ocrOnlyCheckbox)
.addComponent(limitedOcrCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 288, javax.swing.GroupLayout.PREFERRED_SIZE))))
.addContainerGap())))
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@ -305,25 +354,48 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
.addGap(7, 7, 7)
.addComponent(titleLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(listsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 41, Short.MAX_VALUE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(languagesLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(listsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(languagesLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(languagesValLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(encodingsLabel)
.addComponent(keywordSearchEncodings))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(ocrCheckBox)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(ocrOnlyCheckbox)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(limitedOcrCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addContainerGap())
);
}// </editor-fold>//GEN-END:initComponents
private void ocrCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ocrCheckBoxActionPerformed
handleOcrEnabled(ocrCheckBox.isSelected());
firePropertyChange(KeywordSearchOptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_ocrCheckBoxActionPerformed
private void limitedOcrCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_limitedOcrCheckboxActionPerformed
firePropertyChange(KeywordSearchOptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_limitedOcrCheckboxActionPerformed
private void ocrOnlyCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ocrOnlyCheckboxActionPerformed
// TODO add your handling code here:
}//GEN-LAST:event_ocrOnlyCheckboxActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JLabel encodingsLabel;
private javax.swing.JLabel keywordSearchEncodings;
private javax.swing.JLabel languagesLabel;
private javax.swing.JLabel languagesValLabel;
private javax.swing.JCheckBox limitedOcrCheckbox;
private javax.swing.JScrollPane listsScrollPane;
private javax.swing.JTable listsTable;
private javax.swing.JCheckBox ocrCheckBox;
private javax.swing.JCheckBox ocrOnlyCheckbox;
private javax.swing.JLabel titleLabel;
// End of variables declaration//GEN-END:variables
}

65
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java
View File

@ -133,20 +133,14 @@ class KeywordSearchSettings {
ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, key, val);
}
/**
* Save OCR setting to permanent storage
*
* @param enabled Is OCR enabled?
*/
static void setOcrOption(boolean enabled) {
ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, OCR_ENABLED, (enabled ? "true" : "false")); //NON-NLS
}
/**
* Get OCR setting from permanent storage
*
* @return Is OCR enabled?
*
* @deprecated Please use KeywordSearchJobSettings instead.
*/
@Deprecated
static boolean getOcrOption() {
if (ModuleSettings.settingExists(PROPERTIES_OPTIONS, OCR_ENABLED)) {
return ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, OCR_ENABLED).equals("true"); //NON-NLS
@ -155,6 +149,24 @@ class KeywordSearchSettings {
}
}
/**
* Gets the limited OCR flag to indicate if OCR should be limited to larger
* images and images which were extracted from documents.
*
* @return Flag indicating if limited OCR is enabled. True if OCR should be
* limited, false otherwise.
*
* @deprecated Please use KeywordSearchJobSettings instead.
*/
@Deprecated
static boolean getLimitedOcrOption() {
if (ModuleSettings.settingExists(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) {
return ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED).equals("true"); //NON-NLS
} else {
return LIMITED_OCR_ENABLED_DEFAULT;
}
}
static void setShowSnippets(boolean showSnippets) {
ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, SHOW_SNIPPETS, (showSnippets ? "true" : "false")); //NON-NLS
}
@ -246,45 +258,10 @@ class KeywordSearchSettings {
logger.log(Level.INFO, "No configuration for UTF16 found, generating defaults..."); //NON-NLS
KeywordSearchSettings.setStringExtractOption(StringsExtractOptions.EXTRACT_UTF16.toString(), Boolean.TRUE.toString());
}
//setting OCR default (disabled by default)
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, OCR_ENABLED)) {
logger.log(Level.INFO, "No configuration for OCR found, generating defaults..."); //NON-NLS
KeywordSearchSettings.setOcrOption(OCR_ENABLED_DEFAULT);
}
//setting OCR default (disabled by default)
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) {
logger.log(Level.INFO, "No configuration for OCR found, generating defaults..."); //NON-NLS
KeywordSearchSettings.setLimitedOcrOption(LIMITED_OCR_ENABLED_DEFAULT);
}
//setting default Latin-1 Script
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_SCRIPTS, SCRIPT.LATIN_1.name())) {
logger.log(Level.INFO, "No configuration for Scripts found, generating defaults..."); //NON-NLS
ModuleSettings.setConfigSetting(KeywordSearchSettings.PROPERTIES_SCRIPTS, SCRIPT.LATIN_1.name(), Boolean.toString(true));
}
}
/**
* Enables the limiting OCR to be run on larger images and images which were
* extracted from documents.
*
* @param enabled Flag indicating if OCR is enabled.
*/
static void setLimitedOcrOption(boolean enabled) {
ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED, (enabled ? "true" : "false")); //NON-NLS
}
/**
* Gets the limited OCR flag to indicate if OCR should be limited to larger
* images and images which were extracted from documents.
*
* @return Flag indicating if limited OCR is enabled. True if OCR should be
* limited, false otherwise..
*/
static boolean getLimitedOcrOption() {
if (ModuleSettings.settingExists(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) {
return ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED).equals("true"); //NON-NLS
} else {
return LIMITED_OCR_ENABLED_DEFAULT;
}
}
}

14
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -466,18 +466,4 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
throw new TskCoreException(ex.getCause().getMessage(), ex);
}
}
/**
* A flag to enable or disable OCR on all future text indexing. Also sets the
* the "Limited OCR" functionality accordingly.
*
* @param state Boolean flag to enable/disable OCR. Set to True to enable
* OCR, or False to disable it.
*/
@Override
public void changeOcrState(boolean state) {
KeywordSearchSettings.setOcrOption(state);
KeywordSearchSettings.setLimitedOcrOption(state);
}
}

35
NEWS.txt
View File

@ -5,12 +5,18 @@ Data Source Management:
- The main tree viewer can be configured to group by person and host.
OS Accounts:
- Operating System (OS) accounts and realms are their own data types and not generic artifacts.
- Operating System (OS) accounts and realms are their own data types and no longer generic artifacts.
- OS Accounts are created for Windows accounts found in the registry. Domain-scoped realms are not fully detected yet.
- NTFS files are associated with OS Accounts by SID.
- The Recent Activity module associates artifacts with OS Accounts based on SID or path of database. Other modules still need to be updated.
- OS accounts appear in a dedicated sub-tree of the main tree view and their properties can be viewed in the results view.
- A new content viewer in the lower right area of the main window was built to display OS account data for the item selected in the result view.
- A new content viewer in the lower right area of the main window was built to display OS account data for the item selected in the results view.
Analysis Result and Data Artifacts
- All modules make either Analysis Results or Data Artifacts instead of “Blackboard Artifacts.”
- New “Analysis Result” content viewer shows the results for a given file and its score.
- The tabular results viewer shows an icon for the aggregate score of a file.
- The tree organizes results into "Analysis Results" and "Data Artifacts" instead of simply “Results.”
Discovery UI:
- Domain categorization and account types are displayed in Domain Discovery results.
@ -22,17 +28,40 @@ Ingest Modules:
- Parsing of iLEAPP and aLEAPP output was expanded to create communication relationships which can be displayed in the Communications UI.
- EML email parsing handles EML messages that are attachments (and have their own attachments).
- Domain categorization within Recent Activity can be customized by user-defined rules that can be imported and exported.
- Account IDs and Installed Applications are added to the Central Repository.
- Keyword search can be configured to only do OCR and skip non-OCR files.
Miscellaneous:
- A “Reset Windows” feature was created to help redock windows.
- A case-insensitive wordlist of all words in the keyword search index can be exported as a text document.
- Information from the Data Source Summary panels can be exported as an Excel spreadsheet.
- More artifacts are added to the timeline and artifacts with multiple time-based attributes are mapped to multiple timeline events.
- The Auto Ingest Dashboard is resizable.
- Added option to only perform optical character recognition on certain file types.
- Heap dumps can be saved to a custom location.
- More detailed error messages about encrypted disks when they are added.
- Added file size filter to Ingest Filters.
Performance:
- Keyword search does not make an explicit commit for each report if ingest is running.
- Language ID is performed on a small subset of a file instead of the entire file.
- Recent Activity is more efficient because of TSK changes to file searching (using extension).
- Embedded file extractor module has been made faster by doing file typing in memory and adding extracted files in batches.
- Moved Content Viewers setNode() and isSupported()/isPreferred() code to background threads.
- Moved Data Source Summary Panel population code to background threads.
- Moved Node/Tree queries to background threads.
Bug Fixes:
- Fixed embedded file extractor file name escaping bug.
- Detect VHD files by signature and not extension.
- Fixed iLEAPP path error.
- Content viewers UIs are more consistent.
- Assorted bug fixes are included.
Auto Ingest:
- The Auto Ingest Dashboard is resizable.
- Get thread dumps from AID
- Added beta pause feature that pauses auto ingest for a set amount of time at a scheduled date and time.
---------------- VERSION 4.18.0 --------------
Keyword Search:
- A major upgrade from Solr 4 to Solr 8.6.3. Single user cases continue to use the embedded server.

2
RecentActivity/manifest.mf
View File

@ -1,6 +1,6 @@
Manifest-Version: 1.0
OpenIDE-Module: org.sleuthkit.autopsy.recentactivity/6
OpenIDE-Module-Implementation-Version: 18
OpenIDE-Module-Implementation-Version: 19
OpenIDE-Module-Layer: org/sleuthkit/autopsy/recentactivity/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/recentactivity/Bundle.properties
OpenIDE-Module-Requires:

2
RecentActivity/nbproject/project.xml
View File

@ -60,7 +60,7 @@
<compile-dependency/>
<run-dependency>
<release-version>10</release-version>
<specification-version>10.23</specification-version>
<specification-version>10.24</specification-version>
</run-dependency>
</dependency>
<dependency>

4
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java
View File

@ -1995,7 +1995,7 @@ class ExtractRegistry extends Extract {
} else {
osAccount = optional.get();
if (userName != null && !userName.isEmpty()) {
OsAccountUpdateResult updateResult= accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, userName, null, host);
OsAccountUpdateResult updateResult= accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, userName, domainName.isEmpty() ? null : domainName, host);
osAccount = updateResult.getUpdatedAccount().orElse(osAccount);
}
}
@ -2187,7 +2187,7 @@ class ExtractRegistry extends Extract {
accountMgr.addExtendedOsAccountAttributes(osAccount, attributes);
// update the loginname
accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, loginName, null, host);
accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, loginName, domainName.isEmpty() ? null : domainName, host);
// update other standard attributes - fullname, creationdate
accountMgr.updateStandardOsAccountAttributes(osAccount, fullName, null, null, creationTime);

2
TSKVersion.xml
View File

@ -1,3 +1,3 @@
<project name="TSK_VERSION">
<property name="TSK_VERSION" value="4.10.2"/>
<property name="TSK_VERSION" value="4.11.0"/>
</project>

2
Testing/nbproject/project.xml
View File

@ -47,7 +47,7 @@
<compile-dependency/>
<run-dependency>
<release-version>10</release-version>
<specification-version>10.23</specification-version>
<specification-version>10.24</specification-version>
</run-dependency>
</dependency>
<dependency>

4
docs/doxygen-user/Doxyfile
View File

@ -38,7 +38,7 @@ PROJECT_NAME = "Autopsy User Documentation"
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = 4.18.0
PROJECT_NUMBER = 4.19.0
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
@ -1025,7 +1025,7 @@ GENERATE_HTML = YES
# The default directory is: html.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_OUTPUT = 4.18.0
HTML_OUTPUT = 4.19.0
# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
# generated HTML page (for example: .htm, .php, .asp).

14
docs/doxygen-user/auto_ingest_administration.dox
View File

@ -34,7 +34,15 @@ With the admin file in place, the user can right-click on jobs in each of the ta
\image html AutoIngest/admin_jobs_panel.png
In the Running Jobs tables, the ingest progress can be viewed and the current job can be cancelled. Note that cancellation can take some time.
You can also selectively enable \ref keyword_search_ocr_config "optical character recognition (OCR)" on a case-by-case basis. This will override the normal Keyword Search settings for every job in the case. Jobs in progress will not be affected. To enable OCR for a case, right-click on a job from that case in the Pending Jobs table and select "Enable OCR For This Case".
\image html admin_jobs_ocr1.png
Once enabled, a green checkmark will appear in the OCR column next to every pending job for that case.
\image html admin_jobs_ocr2.png
In the Running Jobs tables, the ingest progress can be viewed and the current job can be cancelled. Note that cancellation can take some time. You can also generate a thread dump if you suspect ingest may be stuck.
\image html AutoIngest/admin_jobs_cancel.png
@ -42,9 +50,11 @@ In the Completed Jobs table, the user can reprocess a job (generally useful when
\image html AutoIngest/admin_jobs_completed.png
\section auto_ingest_admin_nodes_panel Auto Ingest Nodes Panel
The Nodes panel displays the status of every online auto ingest node. Additionally, an admin can pause or resume a node, or shut down a node entirely (i.e., exit the Autopsy app).
The Nodes panel displays the status of every online auto ingest node. Additionally, an admin can pause or resume a node, generate a thread dump, or shut down a node entirely (i.e., exit the Autopsy app).
\image html AutoIngest/admin_nodes_panel.png

BIN
docs/doxygen-user/images/AutoIngest/admin_jobs_cancel.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 8.1 KiB

After

Width:  |  Height:  |  Size: 16 KiB

BIN
docs/doxygen-user/images/AutoIngest/admin_jobs_ocr1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 91 KiB

BIN
docs/doxygen-user/images/AutoIngest/admin_jobs_ocr2.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 70 KiB

BIN
docs/doxygen-user/images/AutoIngest/admin_jobs_panel.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 24 KiB

BIN
docs/doxygen-user/images/AutoIngest/admin_nodes_panel.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 18 KiB

BIN
docs/doxygen-user/images/solr/solr_comment_out_updateLog.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 55 KiB

BIN
docs/doxygen-user/images/solr/solr_config_autocommit.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 72 KiB

BIN
docs/doxygen-user/images/solr/solr_transaction_log_errors.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 44 KiB

34
docs/doxygen-user/multi-user/installSolr.dox
View File

@ -207,7 +207,7 @@ Solr creates two types of data that need to be backed up:
<ol><li>In a default installation that data is stored in \c "C:\solr-8.6.3\server\solr zoo_data" (assuming that the Solr package ZIP was extracted into \c "C:\solr-8.6.3" directory).</ol>
</ul>
\section Troubleshooting
\section troubleshooting Troubleshooting / Performance Tuning
\subsection install_solr_delayed_start Delayed Start Problems With Large Number Of Solr Collections
@ -287,4 +287,36 @@ Some notes:
</ul>
\subsection solr_commit_tuning Tuning Solr Server Commit Operations
When Autopsy is running in a multi-user cluster environment with multiple auto ingest nodes, it may be beneficial to tune the configuration of Solr "commit" operations.
Solr has two types of "commits" – hard commits and soft commits. These are explained in detail the following link: https://lucidworks.com/post/understanding-transaction-logs-softcommit-and-commit-in-sorlcloud/
In short:
<ul>
<li>Hard commits flush newly indexed documents from RAM to Solr index on disk.
<li>Depending on configuration, hard commits may or may not make the newly indexed documents "visible" to search.
<li>Soft commits do not flush newly indexed documents to disk but they make the newly indexed documents "visible" to search.
</ul>
By default (when using AutopsyConfig) the Solr servers are performing a "hard" commit every 5 minutes and are also making the newly indexed documents "visible" to search. These operations can be costly when performed on a large index which is located on a shared network drive. In this stuation it can be very beneficial to modify Solr configuration (located in \c "SOLR_INSTALLATION_DIECTORY\server\solr\configsets\AutopsyConfig\conf\solrconfig.xml") with the following changes:
<ol>
<li>Modify "hard commits" to still flush the newly created documents every 5 minutes but not make them "visible". This is accomplished by setting “openSearcher" to "false" in the "autoCommit" section of Solr configuration file.
<li>Enable "soft commits" to be performed every 30 minutes thus making the newly indexed documents "visible" to search every 30 minutes. This is accomplished by enabling the "autoSoftCommit" section of Solr configuration file. The downside is that it may take up to 30 minutes to be able to search the latest document. Keep in mind that this only affects the scenario where an examiner is searching a case while the ingest is still ongoing. Autopsy automatically performs a commit after the ingest is complete so all the documents are immediately visible at that time.
</ol>
The following image shows the Solr configuration changes discussed above:
\image html solr_config_autocommit.jpg
Until a hard commit is performed, by default Solr also maintains a transaction log which contains the raw text of every newly indexed document since last hard commit. When the Solr index is located on a network share, this again can be a very costly operation. Transaction logs can be disabled by commenting out the "updateLog" section of Solr configuration file:
\image html solr_comment_out_updateLog.jpg
Keep in mind that this has an unfortunate side effect of creating CommitTracker errors in Solr logs and Solr admin console. These errors can be ignored if transaction log has been intentionally disabled.
\image html solr_transaction_log_errors.jpg
*/

4
docs/doxygen/Doxyfile
View File

@ -38,7 +38,7 @@ PROJECT_NAME = "Autopsy"
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = 4.18.0
PROJECT_NUMBER = 4.19.0
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears a the top of each page and should give viewer a
@ -1066,7 +1066,7 @@ GENERATE_HTML = YES
# The default directory is: html.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_OUTPUT = api-docs/4.18.0/
HTML_OUTPUT = api-docs/4.19.0/
# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
# generated HTML page (for example: .htm, .php, .asp).

2
nbproject/project.properties
View File

@ -4,7 +4,7 @@ app.title=Autopsy
### lowercase version of above
app.name=${branding.token}
### if left unset, version will default to today's date
app.version=4.18.0
app.version=4.19.0
### build.type must be one of: DEVELOPMENT, RELEASE
#build.type=RELEASE
build.type=DEVELOPMENT

2
thunderbirdparser/nbproject/project.xml
View File

@ -54,7 +54,7 @@
<compile-dependency/>
<run-dependency>
<release-version>10</release-version>
<specification-version>10.23</specification-version>
<specification-version>10.24</specification-version>
</run-dependency>
</dependency>
<dependency>

2
unix_setup.sh
View File

@ -5,7 +5,7 @@
# NOTE: update_sleuthkit_version.pl updates this value and relies
# on it keeping the same name and whitespace. Don't change it.
TSK_VERSION=4.10.2
TSK_VERSION=4.11.0
# In the beginning...