",
+ ContentViewerHtmlStyles.getIndentedClassName(),
+ ContentViewerHtmlStyles.getTextClassName(),
+ Bundle.MetadataWorker_doInBackground_noDataMsg()));
+ } else {
+ startTable(sb);
- /*
+ for (String str : istatStrings) {
+ addMonospacedRow(sb, str);
+
+ /*
* Very long results can cause the UI to hang before
* displaying, so truncate the results if necessary.
- */
- if (sb.length() > 50000) {
- addMonospacedRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.nodeText.truncated"));
- break;
+ */
+ if (sb.length() > 50000) {
+ addMonospacedRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.nodeText.truncated"));
+ break;
+ }
}
- }
- endTable(sb);
+ endTable(sb);
+ }
}
} else {
@@ -461,11 +470,10 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer {
// Add all the data source paths to the "Local Path" value cell.
String[] imagePaths = image.getPaths();
-
if (imagePaths.length > 0) {
addRowWithMultipleValues(sb,
- NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.localPath"),
- imagePaths);
+ NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.localPath"),
+ imagePaths);
} else {
addRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.localPath"),
NbBundle.getMessage(this.getClass(), "Metadata.nodeText.none"));
diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java
index 3ab02ed4d7..4eef25539f 100644
--- a/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java
+++ b/Core/src/org/sleuthkit/autopsy/contentviewers/osaccount/OsAccountDataPanel.java
@@ -42,6 +42,7 @@ import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.contentviewers.layout.ContentViewerDefaults;
import org.sleuthkit.autopsy.contentviewers.osaccount.SectionData.RowData;
+import org.sleuthkit.autopsy.coreutils.TimeZoneUtils;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.Host;
@@ -190,12 +191,8 @@ public class OsAccountDataPanel extends JPanel {
data.addData(Bundle.OsAccountDataPanel_basic_type(),
account.getOsAccountType().isPresent() ? account.getOsAccountType().get().getName() : "");
- Optional
crTime = account.getCreationTime();
- if (crTime.isPresent()) {
- data.addData(Bundle.OsAccountDataPanel_basic_creationDate(), DATE_FORMAT.format(new Date(crTime.get() * 1000)));
- } else {
- data.addData(Bundle.OsAccountDataPanel_basic_creationDate(), "");
- }
+ Optional crTime = account.getCreationTime();
+ data.addData(Bundle.OsAccountDataPanel_basic_creationDate(), crTime.isPresent() ? TimeZoneUtils.getFormattedTime(crTime.get()) : "");
return data;
}
diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java
index 75a6eb971b..ade29e2cfa 100755
--- a/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java
+++ b/Core/src/org/sleuthkit/autopsy/datamodel/OsAccounts.java
@@ -207,7 +207,7 @@ public final class OsAccounts implements AutopsyVisitableItem {
Bundle.OsAccounts_accountRealmNameProperty_name(),
Bundle.OsAccounts_accountRealmNameProperty_displayName(),
Bundle.OsAccounts_accountRealmNameProperty_desc(),
- ""));
+ realmNames.get(0)));
}
}
}
@@ -308,7 +308,7 @@ public final class OsAccounts implements AutopsyVisitableItem {
Optional creationTimeValue = account.getCreationTime();
String timeDisplayStr
- = creationTimeValue.isPresent() ? TimeZoneUtils.getFormattedTime(creationTimeValue.get() * 1000) : "";
+ = creationTimeValue.isPresent() ? TimeZoneUtils.getFormattedTime(creationTimeValue.get()) : "";
propertiesSet.put(new NodeProperty<>(
Bundle.OsAccounts_createdTimeProperty_name(),
diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED
index e9f767cffe..e5e9f22a46 100644
--- a/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED
+++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/Bundle.properties-MERGED
@@ -24,6 +24,7 @@ DateFilterPanel.invalidRange.text=Range or Only Last must be selected.
DateFilterPanel.startAfterEnd.text=Start date should be before the end date when both are enabled.
DateFilterPanel.startOrEndNeeded.text=A start or end date must be specified to use the range filter.
DiscoveryDialog.name.text=Discovery
+DiscoveryExtractAction.title.extractFiles.text=Extract File
DiscoveryTopComponent.additionalFilters.text=;
DiscoveryTopComponent.cancelButton.text=Cancel Search
DiscoveryTopComponent.domainSearch.text=Type: Domain
diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DiscoveryExtractAction.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/DiscoveryExtractAction.java
index 7965878017..59ff07b86b 100644
--- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DiscoveryExtractAction.java
+++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DiscoveryExtractAction.java
@@ -1,7 +1,7 @@
/*
* Autopsy
*
- * Copyright 2019 Basis Technology Corp.
+ * Copyright 2019-2021 Basis Technology Corp.
* Contact: carrier sleuthkit org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,8 +40,9 @@ final class DiscoveryExtractAction extends AbstractAction {
*
* @param selectedFiles The files to extract from the current case.
*/
+ @NbBundle.Messages({"DiscoveryExtractAction.title.extractFiles.text=Extract File"})
DiscoveryExtractAction(Collection selectedFiles) {
- super(NbBundle.getMessage(DiscoveryExtractAction.class, "DiscoveryExtractAction.title.extractFiles.text"));
+ super(Bundle.DiscoveryExtractAction_title_extractFiles_text());
files.addAll(selectedFiles);
}
diff --git a/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java b/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java
index 7d197a4d11..6c2997ccec 100644
--- a/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java
+++ b/Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java
@@ -106,14 +106,4 @@ public interface KeywordSearchService extends Closeable {
* @throws KeywordSearchServiceException if unable to delete.
*/
void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException;
-
- /**
- * A flag to enable or disable OCR on all future text indexing.
- *
- * @param state Boolean flag to enable/disable OCR. Set to True to enable
- * OCR, or False to disable it.
- */
- @Beta
- void changeOcrState(boolean state);
-
}
diff --git a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java
index 1fe5964686..99a1920818 100755
--- a/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java
+++ b/Core/src/org/sleuthkit/autopsy/modules/pictureanalyzer/impls/EXIFProcessor.java
@@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
- * Copyright 2011-2020 Basis Technology Corp.
+ * Copyright 2011-2021 Basis Technology Corp.
* Contact: carrier sleuthkit org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -49,7 +49,6 @@ import org.sleuthkit.autopsy.modules.pictureanalyzer.PictureAnalyzerIngestModule
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF;
-import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_USER_CONTENT_SUSPECTED;
import org.sleuthkit.datamodel.BlackboardAttribute;
import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.Content;
@@ -150,7 +149,11 @@ public class EXIFProcessor implements PictureProcessor {
if (!attributes.isEmpty() && !blackboard.artifactExists(file, TSK_METADATA_EXIF, attributes)) {
- final BlackboardArtifact exifArtifact = file.newDataArtifact(new BlackboardArtifact.Type(TSK_METADATA_EXIF), attributes);
+ final BlackboardArtifact exifArtifact = (file.newAnalysisResult(
+ BlackboardArtifact.Type.TSK_METADATA_EXIF,
+ Score.SCORE_NONE,
+ null, null, null,
+ attributes)).getAnalysisResult();
final BlackboardArtifact userSuspectedArtifact = file.newAnalysisResult(
BlackboardArtifact.Type.TSK_USER_CONTENT_SUSPECTED, Score.SCORE_UNKNOWN, null, null, null,
diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/TextExtractor.java b/Core/src/org/sleuthkit/autopsy/textextractors/TextExtractor.java
index 69b98cdd12..e6c0ddcaba 100644
--- a/Core/src/org/sleuthkit/autopsy/textextractors/TextExtractor.java
+++ b/Core/src/org/sleuthkit/autopsy/textextractors/TextExtractor.java
@@ -22,6 +22,8 @@ import java.io.Reader;
import java.util.Collections;
import java.util.Map;
import org.openide.util.Lookup;
+import org.sleuthkit.autopsy.textextractors.configs.ImageConfig;
+import org.sleuthkit.datamodel.AbstractFile;
/**
* Extracts the text out of Content instances and exposes them as a Reader.
@@ -63,10 +65,10 @@ public interface TextExtractor {
default void setExtractionSettings(Lookup context) {
//no-op by default
}
-
+
/**
* Retrieves content metadata, if any.
- *
+ *
* @return Metadata as key -> value map
*/
default Map getMetadata() {
@@ -74,7 +76,17 @@ public interface TextExtractor {
}
/**
- * System level exception for reader initialization.
+ * Returns true if this text extractor, based on the provided settings, will
+ * perform ocr.
+ *
+ * @return True if will perform OCR.
+ */
+ default boolean willUseOCR() {
+ return false;
+ }
+
+ /**
+ * System level exception for reader initialization.
*/
public class InitReaderException extends Exception {
diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java
index 9540f72271..470c0f8cf4 100644
--- a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java
+++ b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java
@@ -18,7 +18,6 @@
*/
package org.sleuthkit.autopsy.textextractors;
-import com.google.common.collect.ImmutableList;
import com.google.common.io.CharSource;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.File;
@@ -71,12 +70,15 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Set;
+import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.pdf.PDFParserConfig.OCR_STRATEGY;
import org.sleuthkit.autopsy.coreutils.ExecUtil.HybridTerminator;
-import org.sleuthkit.datamodel.TskData;
+import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
/**
* Extracts text from Tika supported content. Protects against Tika parser hangs
@@ -86,8 +88,8 @@ final class TikaTextExtractor implements TextExtractor {
//Mimetype groups to aassist extractor implementations in ignoring binary and
//archive files.
- private static final List BINARY_MIME_TYPES
- = ImmutableList.of(
+ private static final Set BINARY_MIME_TYPES
+ = ImmutableSet.of(
//ignore binary blob data, for which string extraction will be used
"application/octet-stream", //NON-NLS
"application/x-msdownload"); //NON-NLS
@@ -96,8 +98,8 @@ final class TikaTextExtractor implements TextExtractor {
* generally text extractors should ignore archives and let unpacking
* modules take care of them
*/
- private static final List ARCHIVE_MIME_TYPES
- = ImmutableList.of(
+ private static final Set ARCHIVE_MIME_TYPES
+ = ImmutableSet.of(
//ignore unstructured binary and compressed data, for which string extraction or unzipper works better
"application/x-7z-compressed", //NON-NLS
"application/x-ace-compressed", //NON-NLS
@@ -134,22 +136,36 @@ final class TikaTextExtractor implements TextExtractor {
// Used to log to the tika file that is why it uses the java.util.logging.logger class instead of the Autopsy one
private static final java.util.logging.Logger TIKA_LOGGER = java.util.logging.Logger.getLogger("Tika"); //NON-NLS
private static final Logger AUTOPSY_LOGGER = Logger.getLogger(TikaTextExtractor.class.getName());
- private static final int LIMITED_OCR_SIZE_MIN = 100 * 1024;
+
private final ThreadFactory tikaThreadFactory
= new ThreadFactoryBuilder().setNameFormat("tika-reader-%d").build();
private final ExecutorService executorService = Executors.newSingleThreadExecutor(tikaThreadFactory);
private static final String SQLITE_MIMETYPE = "application/x-sqlite3";
private final AutoDetectParser parser = new AutoDetectParser();
+ private final FileTypeDetector fileTypeDetector;
private final Content content;
private boolean tesseractOCREnabled;
- private boolean limitedOCREnabled;
private static final String TESSERACT_DIR_NAME = "Tesseract-OCR"; //NON-NLS
private static final String TESSERACT_EXECUTABLE = "tesseract.exe"; //NON-NLS
private static final File TESSERACT_PATH = locateTesseractExecutable();
private String languagePacks = formatLanguagePacks(PlatformUtil.getOcrLanguagePacks());
private static final String TESSERACT_OUTPUT_FILE_NAME = "tess_output"; //NON-NLS
+
+ // documents where OCR is performed
+ private static final ImmutableSet OCR_DOCUMENTS = ImmutableSet.of(
+ "application/pdf",
+ "application/msword",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.ms-powerpoint",
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+ "application/vnd.ms-excel",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+ );
+
+ private static final String IMAGE_MIME_TYPE_PREFIX = "image/";
+
private Map metadataMap;
private ProcessTerminator processTerminator;
@@ -162,18 +178,62 @@ final class TikaTextExtractor implements TextExtractor {
TikaTextExtractor(Content content) {
this.content = content;
+
+ FileTypeDetector detector = null;
+ try {
+ detector = new FileTypeDetector();
+ } catch (FileTypeDetector.FileTypeDetectorInitException ex) {
+ TIKA_LOGGER.log(Level.SEVERE, "Unable to instantiate a file type detector", ex);
+ }
+ this.fileTypeDetector = detector;
}
/**
- * If Tesseract has been installed and is set to be used through
- * configuration, then ocr is enabled. OCR can only currently be run on 64
- * bit Windows OS.
+ * Obtains the mime type of the file using a FileTypeDetector with the
+ * file's mime type as fallback if the FileTypeDetector is not instantiated.
+ * If no mime type present, MimeTypes.OCTET_STREAM is returned.
*
- * @return Flag indicating if OCR is set to be used.
+ * @param file The abstract file instance.
+ *
+ * @return The mime type or MimeTypes.OCTET_STREAM if the mime type cannot
+ * be determined.
*/
- private boolean ocrEnabled() {
- return TESSERACT_PATH != null && tesseractOCREnabled
- && PlatformUtil.isWindowsOS() == true && PlatformUtil.is64BitOS();
+ private String getMimeType(AbstractFile file) {
+ String mimeType = MimeTypes.OCTET_STREAM;
+ if (fileTypeDetector != null) {
+ mimeType = fileTypeDetector.getMIMEType(file);
+ } else if (file.getMIMEType() != null) {
+ mimeType = file.getMIMEType();
+ }
+
+ return mimeType.trim().toLowerCase();
+ }
+
+ @Override
+ public boolean willUseOCR() {
+ if (!isOcrSupported() || (!(content instanceof AbstractFile))) {
+ return false;
+ }
+
+ String mimeType = getMimeType((AbstractFile) content);
+ // in order to ocr, it needs to either be an image or a document with embedded content
+ return mimeType.startsWith(IMAGE_MIME_TYPE_PREFIX) || OCR_DOCUMENTS.contains(mimeType);
+ }
+
+ /**
+ * Whether or not OCR is supported in environment.
+ *
+ * @return True if OCR is supported.
+ */
+ private boolean isOcrSupported() {
+ // If Tesseract has been installed and is set to be used through
+ // configuration, then ocr is enabled. OCR can only currently be run on 64
+ // bit Windows OS.
+ return TESSERACT_PATH != null
+ && tesseractOCREnabled
+ && PlatformUtil.isWindowsOS()
+ && PlatformUtil.is64BitOS()
+ && isSupported();
}
/**
@@ -195,33 +255,31 @@ final class TikaTextExtractor implements TextExtractor {
// Only abstract files are supported, see isSupported()
final AbstractFile file = ((AbstractFile) content);
- // This mime type must be non-null, see isSupported()
- final String mimeType = file.getMIMEType();
+
+ String mimeType = getMimeType(file);
// Handle images seperately so the OCR task can be cancelled.
// See JIRA-4519 for the need to have cancellation in the UI and ingest.
- if (ocrEnabled() && mimeType.toLowerCase().startsWith("image/") && useOcrOnFile(file)) {
+ if (isOcrSupported() && mimeType.startsWith(IMAGE_MIME_TYPE_PREFIX)) {
InputStream imageOcrStream = performOCR(file);
return new InputStreamReader(imageOcrStream, Charset.forName("UTF-8"));
}
// Set up Tika
final InputStream stream = new ReadContentInputStream(content);
- final ParseContext parseContext = new ParseContext();
+ final ParseContext parseContext = new ParseContext();
// Documents can contain other documents. By adding
// the parser back into the context, Tika will recursively
// parse embedded documents.
parseContext.set(Parser.class, parser);
-
// Use the more memory efficient Tika SAX parsers for DOCX and
// PPTX files (it already uses SAX for XLSX).
OfficeParserConfig officeParserConfig = new OfficeParserConfig();
officeParserConfig.setUseSAXPptxExtractor(true);
officeParserConfig.setUseSAXDocxExtractor(true);
parseContext.set(OfficeParserConfig.class, officeParserConfig);
-
- if (ocrEnabled() && useOcrOnFile(file)) {
+ if (isOcrSupported()) {
// Configure OCR for Tika if it chooses to run OCR
// during extraction
TesseractOCRConfig ocrConfig = new TesseractOCRConfig();
@@ -234,10 +292,10 @@ final class TikaTextExtractor implements TextExtractor {
// Configure how Tika handles OCRing PDFs
PDFParserConfig pdfConfig = new PDFParserConfig();
- // This stategy tries to pick between OCRing a page in the
+ // This stategy tries to pick between OCRing a page in the
// PDF and doing text extraction. It makes this choice by
// first running text extraction and then counting characters.
- // If there are too few characters or too many unmapped
+ // If there are too few characters or too many unmapped
// unicode characters, it'll run the entire page through OCR
// and take that output instead. See JIRA-6938
pdfConfig.setOcrStrategy(OCR_STRATEGY.AUTO);
@@ -347,22 +405,6 @@ final class TikaTextExtractor implements TextExtractor {
}
}
- /**
- * Method to indicate if OCR should be performed on this image file. Checks
- * to see if the limited OCR setting is enabled. If it is it will also check
- * that one of the limiting factors is true.
- *
- * @param file The AbstractFile which OCR might be performed on.
- * @param boolean The configuration setting which indicates if limited OCR
- * is enabled in Keyword Search.
- *
- * @return True if limited OCR is not enabled or the image is greater than
- * 100KB in size or the image is a derived file.
- */
- private boolean useOcrOnFile(AbstractFile file) {
- return !limitedOCREnabled || file.getSize() > LIMITED_OCR_SIZE_MIN || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED;
- }
-
/**
* Wraps the creation of a TikaReader into a Future so that it can be
* cancelled.
@@ -552,7 +594,6 @@ final class TikaTextExtractor implements TextExtractor {
ImageConfig configInstance = context.lookup(ImageConfig.class);
if (configInstance != null) {
this.tesseractOCREnabled = configInstance.getOCREnabled();
- this.limitedOCREnabled = configInstance.getLimitedOCREnabled();
if (Objects.nonNull(configInstance.getOCRLanguages())) {
this.languagePacks = formatLanguagePacks(configInstance.getOCRLanguages());
diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java b/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java
index fc4f27add1..39fb7db959 100755
--- a/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java
+++ b/Core/src/org/sleuthkit/autopsy/textextractors/configs/ImageConfig.java
@@ -35,7 +35,6 @@ public class ImageConfig {
private static final int OCR_TIMEOUT_SECONDS = 30 * 60;
private boolean OCREnabled = false;
- private boolean limitedOCREnabled = false;
private List ocrLanguages = null;
private final TimedProcessTerminator ocrTimedTerminator = new TimedProcessTerminator(OCR_TIMEOUT_SECONDS);
@@ -49,16 +48,6 @@ public class ImageConfig {
this.OCREnabled = enabled;
}
- /**
- * Enables the limiting OCR to be run on larger images and images which were
- * extracted from documents.
- *
- * @param enabled Flag indicating if OCR is enabled.
- */
- public void setLimitedOCREnabled(boolean enabled) {
- this.limitedOCREnabled = enabled;
- }
-
/**
* Gets the OCR flag that has been set. By default this flag is turned off.
*
@@ -67,7 +56,7 @@ public class ImageConfig {
public boolean getOCREnabled() {
return this.OCREnabled;
}
-
+
/**
* Sets languages for OCR. Can be null.
*
@@ -98,15 +87,4 @@ public class ImageConfig {
public ProcessTerminator getOCRTimeoutTerminator() {
return ocrTimedTerminator;
}
-
- /**
- * Gets the limited OCR flag to indicate if OCR should be limited to larger
- * images and images which were extracted from documents.
- *
- * @return Flag indicating if limited OCR is enabled. True if OCR should be
- * limited, false otherwise..
- */
- public boolean getLimitedOCREnabled() {
- return limitedOCREnabled;
- }
}
diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java
index 62ca12e217..b5424d55a0 100755
--- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java
+++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java
@@ -1270,7 +1270,7 @@ public class CentralRepoDatamodelTest extends TestCase {
// We expect 10 - the custom type is disabled
// Note: this test will need to be updated based on the current default items defined in the correlation_types table
- assertTrue("getDefinedCorrelationTypes returned " + types.size() + " enabled entries - expected 27", types.size() == 27);
+ assertTrue("getDefinedCorrelationTypes returned " + types.size() + " enabled entries - expected 29", types.size() == 29);
} catch (CentralRepoException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex.getMessage());
@@ -1282,7 +1282,7 @@ public class CentralRepoDatamodelTest extends TestCase {
// We expect 10 - the custom type is not supported
// Note: this test will need to be updated based on the current default items defined in the correlation_types table
- assertTrue("getDefinedCorrelationTypes returned " + types.size() + " supported entries - expected 27", types.size() == 27);
+ assertTrue("getDefinedCorrelationTypes returned " + types.size() + " supported entries - expected 29", types.size() == 29);
} catch (CentralRepoException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex.getMessage());
diff --git a/Experimental/nbproject/project.xml b/Experimental/nbproject/project.xml
index 41343b8d23..34734e4954 100644
--- a/Experimental/nbproject/project.xml
+++ b/Experimental/nbproject/project.xml
@@ -144,7 +144,7 @@
10
- 10.23
+ 10.24
diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
index c7eb4111bf..361a071f0b 100644
--- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
+++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
@@ -105,11 +105,11 @@ import org.sleuthkit.autopsy.ingest.IngestModuleError;
import org.sleuthkit.autopsy.ingest.IngestStream;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException;
import org.sleuthkit.autopsy.keywordsearch.Server;
-import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
+import org.sleuthkit.autopsy.keywordsearch.KeywordSearchJobSettings;
/**
* An auto ingest manager is responsible for processing auto ingest jobs defined
@@ -2295,15 +2295,6 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
currentJob.setProcessingStage(AutoIngestJob.Stage.UPDATING_SHARED_CONFIG, Date.from(Instant.now()));
new SharedConfiguration().downloadConfiguration();
}
-
- // update the OCR enabled/disabled setting
- if (currentJob.getOcrEnabled()) {
- sysLogger.log(Level.INFO, "Enabling OCR for job {0}", currentJob.getManifest().getFilePath());
- } else {
- sysLogger.log(Level.INFO, "Disabling OCR for job {0}", currentJob.getManifest().getFilePath());
- }
- KeywordSearchService kwsService = Lookup.getDefault().lookup(KeywordSearchService.class);
- kwsService.changeOcrState(currentJob.getOcrEnabled());
}
/**
@@ -2753,7 +2744,8 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
sysLogger.log(Level.WARNING, "Cancellation while waiting for data source processor for {0}", manifestPath);
jobLogger.logDataSourceProcessorCancelled();
}
- }
+ }
+
/**
* Analyzes the data source content returned by the data source
@@ -2798,6 +2790,20 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
throw new AnalysisStartupException("Error(s) in ingest job settings");
}
+ // update the OCR enabled/disabled setting
+ if (currentJob.getOcrEnabled()) {
+ sysLogger.log(Level.INFO, "Enabling OCR for job {0}", currentJob.getManifest().getFilePath());
+ } else {
+ sysLogger.log(Level.INFO, "Disabling OCR for job {0}", currentJob.getManifest().getFilePath());
+ }
+
+ // find the KeywordSearchJobSettings instance in the templates, and if present, set ocr enabled
+ ingestJobSettings.getIngestModuleTemplates().stream()
+ .filter(template -> template.getModuleSettings() instanceof KeywordSearchJobSettings)
+ .map(template -> (KeywordSearchJobSettings) template.getModuleSettings())
+ .findFirst()
+ .ifPresent((keywordJobSettings) -> keywordJobSettings.setOCREnabled(currentJob.getOcrEnabled()));
+
ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings);
ingestJob = ingestJobStartResult.getJob();
@@ -2861,6 +2867,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
currentJob.setIngestJob(null);
}
}
+
/**
* Gather metrics to store in auto ingest job nodes. A SleuthkitCase
diff --git a/ImageGallery/nbproject/project.xml b/ImageGallery/nbproject/project.xml
index 61f94648d4..0c5fd31fca 100644
--- a/ImageGallery/nbproject/project.xml
+++ b/ImageGallery/nbproject/project.xml
@@ -127,7 +127,7 @@
10
- 10.23
+ 10.24
diff --git a/KeywordSearch/manifest.mf b/KeywordSearch/manifest.mf
index 608eaffd6c..0e947e073b 100644
--- a/KeywordSearch/manifest.mf
+++ b/KeywordSearch/manifest.mf
@@ -1,7 +1,7 @@
Manifest-Version: 1.0
AutoUpdate-Show-In-Client: true
OpenIDE-Module: org.sleuthkit.autopsy.keywordsearch/6
-OpenIDE-Module-Implementation-Version: 22
+OpenIDE-Module-Implementation-Version: 23
OpenIDE-Module-Install: org/sleuthkit/autopsy/keywordsearch/Installer.class
OpenIDE-Module-Layer: org/sleuthkit/autopsy/keywordsearch/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/keywordsearch/Bundle.properties
diff --git a/KeywordSearch/nbproject/project.xml b/KeywordSearch/nbproject/project.xml
index c94da5eb38..1a1760e572 100644
--- a/KeywordSearch/nbproject/project.xml
+++ b/KeywordSearch/nbproject/project.xml
@@ -128,7 +128,7 @@
10
- 10.23
+ 10.24
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
index 7dfee92048..831379e537 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
@@ -319,5 +319,6 @@ ExtractedContentPanel.pageTotalLabel.text=-
ExtractedContentPanel.pageOfLabel.text=of
ExtractedContentPanel.pageCurLabel.text=-
ExtractedContentPanel.pagesLabel.text=Page:
-KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
-KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text=Only process images which are over 100KB in size or extracted from a document (Beta)
+KeywordSearchJobSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
+KeywordSearchJobSettingsPanel.limitedOcrCheckbox.text=Only process PDFs, MS Office docs and images which are over 100KB in size or extracted from another file (Beta)
+KeywordSearchJobSettingsPanel.ocrOnlyCheckbox.text=Only index text extracted using OCR
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED
index 2839859347..5d09c26249 100755
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED
@@ -402,8 +402,9 @@ ExtractedContentPanel.pageTotalLabel.text=-
ExtractedContentPanel.pageOfLabel.text=of
ExtractedContentPanel.pageCurLabel.text=-
ExtractedContentPanel.pagesLabel.text=Page:
-KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
-KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text=Only process images which are over 100KB in size or extracted from a document (Beta)
+KeywordSearchJobSettingsPanel.ocrCheckBox.text=Enable Optical Character Recognition (OCR)
+KeywordSearchJobSettingsPanel.limitedOcrCheckbox.text=Only process images which are over 100KB in size or extracted from a document (Beta)
+KeywordSearchJobSettingsPanel.ocrOnlyCheckbox.text=Only index text extracted using OCR
TextZoomPanel.zoomInButton.text=
TextZoomPanel.zoomOutButton.text=
TextZoomPanel.zoomResetButton.text=Reset
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties
index 74f646d82b..9a7b05f8cb 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties
@@ -388,3 +388,4 @@ SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata\u306b\u306f\u
SolrSearchService.exceptionMessage.noIndexMetadata=\u30b1\u30fc\u30b9\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u30fc\u304b\u3089\u6b21\u306eIndexMetaData\u3092\u4f5c\u6210\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\: {0}
SolrSearchService.exceptionMessage.unableToDeleteCollection=\u30b3\u30ec\u30af\u30b7\u30e7\u30f3{0}\u3092\u524a\u9664\u3067\u304d\u307e\u305b\u3093
TextZoomPanel.zoomResetButton.text=\u30ea\u30bb\u30c3\u30c8
+KeywordSearchJobSettingsPanel.ocrCheckBox.text=OCR\u6587\u5b57\u8a8d\u8b58\u3092\u6709\u52b9\u306b\u3059\u308b
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form
index 550ad5d442..4a0cadaefd 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form
@@ -23,25 +23,25 @@
-
+
-
-
-
-
-
+
+
+
+
+
+
+
+
-
+
-
-
-
@@ -49,7 +49,12 @@
-
+
+
+
+
+
+
@@ -58,22 +63,18 @@
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
@@ -90,10 +91,6 @@
-
-
-
-
@@ -279,25 +276,5 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java
index b9687a32c2..ccd1de71da 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java
@@ -50,13 +50,9 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
private void activateWidgets() {
skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown());
showSnippetsCB.setSelected(KeywordSearchSettings.getShowSnippets());
- ocrCheckBox.setSelected(KeywordSearchSettings.getOcrOption());
- limitedOcrCheckbox.setSelected(KeywordSearchSettings.getLimitedOcrOption());
boolean ingestRunning = IngestManager.getInstance().isIngestRunning();
ingestWarningLabel.setVisible(ingestRunning);
skipNSRLCheckBox.setEnabled(!ingestRunning);
- ocrCheckBox.setEnabled(!ingestRunning);
- limitedOcrCheckbox.setEnabled(ocrCheckBox.isSelected() && !ingestRunning);
setTimeSettingEnabled(!ingestRunning);
final UpdateFrequency curFreq = KeywordSearchSettings.getUpdateFrequency();
@@ -111,8 +107,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
showSnippetsCB = new javax.swing.JCheckBox();
timeRadioButton5 = new javax.swing.JRadioButton();
ingestWarningLabel = new javax.swing.JLabel();
- ocrCheckBox = new javax.swing.JCheckBox();
- limitedOcrCheckbox = new javax.swing.JCheckBox();
skipNSRLCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text")); // NOI18N
skipNSRLCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText")); // NOI18N
@@ -186,20 +180,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
ingestWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/hashdatabase/warning16.png"))); // NOI18N
ingestWarningLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.ingestWarningLabel.text")); // NOI18N
- ocrCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.ocrCheckBox.text")); // NOI18N
- ocrCheckBox.addActionListener(new java.awt.event.ActionListener() {
- public void actionPerformed(java.awt.event.ActionEvent evt) {
- ocrCheckBoxActionPerformed(evt);
- }
- });
-
- limitedOcrCheckbox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.limitedOcrCheckbox.text")); // NOI18N
- limitedOcrCheckbox.addActionListener(new java.awt.event.ActionListener() {
- public void actionPerformed(java.awt.event.ActionEvent evt) {
- limitedOcrCheckboxActionPerformed(evt);
- }
- });
-
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
@@ -207,13 +187,15 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addComponent(ingestWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
+ .addGroup(layout.createSequentialGroup()
+ .addGap(16, 16, 16)
+ .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+ .addComponent(skipNSRLCheckBox)
+ .addComponent(showSnippetsCB))
+ .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addGroup(layout.createSequentialGroup()
- .addComponent(settingsLabel)
- .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
- .addComponent(settingsSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 326, javax.swing.GroupLayout.PREFERRED_SIZE))
+ .addComponent(ingestWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addGroup(layout.createSequentialGroup()
.addComponent(informationLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
@@ -221,14 +203,15 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addGroup(layout.createSequentialGroup()
.addGap(16, 16, 16)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addComponent(skipNSRLCheckBox)
- .addComponent(showSnippetsCB)
- .addComponent(ocrCheckBox)
.addGroup(layout.createSequentialGroup()
.addComponent(filesIndexedLabel)
.addGap(18, 18, 18)
.addComponent(filesIndexedValue))
.addComponent(frequencyLabel)
+ .addGroup(layout.createSequentialGroup()
+ .addComponent(chunksLabel)
+ .addGap(18, 18, 18)
+ .addComponent(chunksValLabel))
.addGroup(layout.createSequentialGroup()
.addGap(16, 16, 16)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@@ -236,16 +219,13 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addComponent(timeRadioButton1)
.addComponent(timeRadioButton3)
.addComponent(timeRadioButton4)
- .addComponent(timeRadioButton5)))
- .addGroup(layout.createSequentialGroup()
- .addComponent(chunksLabel)
- .addGap(18, 18, 18)
- .addComponent(chunksValLabel))
- .addGroup(layout.createSequentialGroup()
- .addGap(16, 16, 16)
- .addComponent(limitedOcrCheckbox)))))
- .addGap(0, 0, Short.MAX_VALUE)))
- .addContainerGap())
+ .addComponent(timeRadioButton5))))))
+ .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
+ .addGroup(layout.createSequentialGroup()
+ .addComponent(settingsLabel)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
+ .addComponent(settingsSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 326, javax.swing.GroupLayout.PREFERRED_SIZE)
+ .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
);
layout.linkSize(javax.swing.SwingConstants.HORIZONTAL, new java.awt.Component[] {chunksLabel, filesIndexedLabel});
@@ -262,10 +242,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(showSnippetsCB)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
- .addComponent(ocrCheckBox)
- .addGap(0, 0, 0)
- .addComponent(limitedOcrCheckbox)
- .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(frequencyLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(timeRadioButton1)
@@ -323,15 +299,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_timeRadioButton4ActionPerformed
- private void ocrCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ocrCheckBoxActionPerformed
- limitedOcrCheckbox.setEnabled(ocrCheckBox.isSelected());
- firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
- }//GEN-LAST:event_ocrCheckBoxActionPerformed
-
- private void limitedOcrCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_limitedOcrCheckboxActionPerformed
- firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
- }//GEN-LAST:event_limitedOcrCheckboxActionPerformed
-
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JLabel chunksLabel;
private javax.swing.JLabel chunksValLabel;
@@ -341,8 +308,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
private javax.swing.JLabel informationLabel;
private javax.swing.JSeparator informationSeparator;
private javax.swing.JLabel ingestWarningLabel;
- private javax.swing.JCheckBox limitedOcrCheckbox;
- private javax.swing.JCheckBox ocrCheckBox;
private javax.swing.JLabel settingsLabel;
private javax.swing.JSeparator settingsSeparator;
private javax.swing.JCheckBox showSnippetsCB;
@@ -360,8 +325,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
KeywordSearchSettings.setSkipKnown(skipNSRLCheckBox.isSelected());
KeywordSearchSettings.setUpdateFrequency(getSelectedTimeValue());
KeywordSearchSettings.setShowSnippets(showSnippetsCB.isSelected());
- KeywordSearchSettings.setOcrOption(ocrCheckBox.isSelected());
- KeywordSearchSettings.setLimitedOcrOption(limitedOcrCheckbox.isSelected());
}
@Override
@@ -412,15 +375,6 @@ class KeywordSearchGlobalSearchSettingsPanel extends javax.swing.JPanel implemen
logger.log(Level.WARNING, "Could not get number of indexed files/chunks"); //NON-NLS
}
- if (!PlatformUtil.isWindowsOS() || !PlatformUtil.is64BitOS()) {
- ocrCheckBox.setText(Bundle.KeywordSearchGlobalSearchSettingsPanel_customizeComponents_windowsOCR());
- ocrCheckBox.setSelected(false);
- ocrCheckBox.setEnabled(false);
- limitedOcrCheckbox.setSelected(false);
- limitedOcrCheckbox.setEnabled(false);
- limitedOcrCheckbox.setText(Bundle.KeywordSearchGlobalSearchSettingsPanel_customizeComponents_windowsLimitedOCR());
- }
-
KeywordSearch.addNumIndexedFilesChangeListener(
new PropertyChangeListener() {
@Override
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
index 034f53af74..d350bfea6d 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
@@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
- * Copyright 2011-2019 Basis Technology Corp.
+ * Copyright 2011-2021 Basis Technology Corp.
* Contact: carrier sleuthkit org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.keywordsearch;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.io.CharSource;
import java.io.IOException;
import java.io.Reader;
@@ -32,6 +33,7 @@ import java.util.HashMap;
import java.util.List;
import static java.util.Locale.US;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.stream.Collectors;
@@ -43,7 +45,6 @@ import org.openide.util.lookup.Lookups;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.ExecUtil.ProcessTerminator;
-import org.sleuthkit.autopsy.coreutils.ExecUtil.TimedProcessTerminator;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
@@ -87,6 +88,8 @@ import org.sleuthkit.datamodel.TskData.FileKnown;
})
public final class KeywordSearchIngestModule implements FileIngestModule {
+ private static final int LIMITED_OCR_SIZE_MIN = 100 * 1024;
+
/**
* generally text extractors should ignore archives and let unpacking
* modules take care of them
@@ -132,19 +135,31 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
"Last-Printed", //NON-NLS
"Creation-Date"); //NON-NLS
- private static final Map METADATA_TYPES_MAP = ImmutableMap.builder()
- .put("Last-Save-Date", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_MODIFIED)
- .put("Last-Author", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_ID)
- .put("Creation-Date", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED)
- .put("Company", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ORGANIZATION)
- .put("Author", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_OWNER)
- .put("Application-Name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME)
- .put("Last-Printed", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LAST_PRINTED_DATETIME)
- .put("Producer", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME)
- .put("Title", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION)
- .put("pdf:PDFVersion", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VERSION)
- .build();
+ private static final Map METADATA_TYPES_MAP = ImmutableMap.builder()
+ .put("Last-Save-Date", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_MODIFIED)
+ .put("Last-Author", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_USER_ID)
+ .put("Creation-Date", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED)
+ .put("Company", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ORGANIZATION)
+ .put("Author", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_OWNER)
+ .put("Application-Name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME)
+ .put("Last-Printed", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LAST_PRINTED_DATETIME)
+ .put("Producer", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME)
+ .put("Title", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION)
+ .put("pdf:PDFVersion", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_VERSION)
+ .build();
+ private static final String IMAGE_MIME_TYPE_PREFIX = "image/";
+
+ // documents where OCR is performed
+ private static final ImmutableSet OCR_DOCUMENTS = ImmutableSet.of(
+ "application/pdf",
+ "application/msword",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.ms-powerpoint",
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+ "application/vnd.ms-excel",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+ );
/**
* Options for this extractor
@@ -206,8 +221,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
* for final statistics at the end of the job.
*
* @param ingestJobId id of ingest job
- * @param fileId id of file
- * @param status ingest status of the file
+ * @param fileId id of file
+ * @param status ingest status of the file
*/
private static void putIngestStatus(long ingestJobId, long fileId, IngestStatus status) {
synchronized (ingestStatus) {
@@ -350,12 +365,31 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
return ProcessResult.OK;
}
+ // if ocr only is enabled and not an ocr file, return
+ Optional extractorOpt = getExtractor(abstractFile);
+
+ String mimeType = fileTypeDetector.getMIMEType(abstractFile).trim().toLowerCase();
+
+ if (settings.isOCREnabled()) {
+ // if ocr only and the extractor is not present or will not perform ocr on this file, continue
+ if (settings.isOCROnly() && (!extractorOpt.isPresent() || !extractorOpt.get().willUseOCR())) {
+ return ProcessResult.OK;
+ }
+
+ // if limited ocr is enabled, the extractor will use ocr, and
+ // the file would not be subject to limited ocr reading, continue
+ if (settings.isLimitedOCREnabled() && extractorOpt.isPresent()
+ && extractorOpt.get().willUseOCR() && !isLimitedOCRFile(abstractFile, mimeType)) {
+ return ProcessResult.OK;
+ }
+ }
+
if (KeywordSearchSettings.getSkipKnown() && abstractFile.getKnown().equals(FileKnown.KNOWN)) {
//index meta-data only
if (context.fileIngestIsCancelled()) {
return ProcessResult.OK;
}
- indexer.indexFile(abstractFile, false);
+ indexer.indexFile(extractorOpt, abstractFile, mimeType, false);
return ProcessResult.OK;
}
@@ -363,7 +397,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return ProcessResult.OK;
}
- indexer.indexFile(abstractFile, true);
+ indexer.indexFile(extractorOpt, abstractFile, mimeType, true);
// Start searching if it hasn't started already
if (!startedSearching) {
@@ -427,6 +461,29 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
initialized = false;
}
+ /**
+ * Returns true if file should have OCR performed on it when limited OCR
+ * setting is specified.
+ *
+ * @param aFile The abstract file.
+ * @param mimeType The file mime type.
+ *
+ * @return True if file should have text extracted when limited OCR setting
+ * is on.
+ */
+ private boolean isLimitedOCRFile(AbstractFile aFile, String mimeType) {
+ if (OCR_DOCUMENTS.contains(mimeType)) {
+ return true;
+ }
+
+ if (mimeType.startsWith(IMAGE_MIME_TYPE_PREFIX)) {
+ return aFile.getSize() > LIMITED_OCR_SIZE_MIN
+ || aFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED;
+ }
+
+ return false;
+ }
+
/**
* Posts inbox message with summary of text_ingested files
*/
@@ -489,6 +546,18 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
}
+ private Optional getExtractor(AbstractFile abstractFile) {
+ ImageConfig imageConfig = new ImageConfig();
+ imageConfig.setOCREnabled(settings.isOCREnabled());
+ ProcessTerminator terminator = () -> context.fileIngestIsCancelled();
+ Lookup extractionContext = Lookups.fixed(imageConfig, terminator);
+ try {
+ return Optional.ofNullable(TextExtractorFactory.getExtractor(abstractFile, extractionContext));
+ } catch (TextExtractorFactory.NoTextExtractorFound ex) {
+ return Optional.empty();
+ }
+ }
+
/**
* File indexer, processes and indexes known/allocated files,
* unknown/unallocated files and directories accordingly
@@ -502,25 +571,26 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
* streaming) from the file Divide the file into chunks and index the
* chunks
*
- * @param aFile file to extract strings from, divide into chunks and
- * index
- * @param extractedMetadata Map that will be populated with the file's metadata.
+ * @param extractorOptional The textExtractor to use with this file or
+ * empty.
+ * @param aFile file to extract strings from, divide into
+ * chunks and index
+ * @param extractedMetadata Map that will be populated with the file's
+ * metadata.
*
* @return true if the file was text_ingested, false otherwise
*
* @throws IngesterException exception thrown if indexing failed
*/
- private boolean extractTextAndIndex(AbstractFile aFile, Map extractedMetadata) throws IngesterException {
- ImageConfig imageConfig = new ImageConfig();
- imageConfig.setOCREnabled(KeywordSearchSettings.getOcrOption());
- imageConfig.setLimitedOCREnabled(KeywordSearchSettings.getLimitedOcrOption());
- ProcessTerminator terminator = () -> context.fileIngestIsCancelled();
- Lookup extractionContext = Lookups.fixed(imageConfig, terminator);
+ private boolean extractTextAndIndex(Optional extractorOptional, AbstractFile aFile,
+ Map extractedMetadata) throws IngesterException {
try {
- TextExtractor extractor = TextExtractorFactory.getExtractor(aFile, extractionContext);
+ if (!extractorOptional.isPresent()) {
+ return false;
+ }
+ TextExtractor extractor = extractorOptional.get();
Reader fileText = extractor.getReader();
-
Reader finalReader;
try {
Map metadata = extractor.getMetadata();
@@ -547,17 +617,17 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
//divide into chunks and index
return Ingester.getDefault().indexText(finalReader, aFile.getId(), aFile.getName(), aFile, context);
- } catch (TextExtractorFactory.NoTextExtractorFound | TextExtractor.InitReaderException ex) {
- //No text extractor found... run the default instead
+ } catch (TextExtractor.InitReaderException ex) {
+ // Text extractor could not be initialized. No text will be extracted.
return false;
}
}
-
+
private void createMetadataArtifact(AbstractFile aFile, Map metadata) {
-
+
String moduleName = KeywordSearchIngestModule.class.getName();
-
- Collection attributes = new ArrayList<>();
+
+ Collection attributes = new ArrayList<>();
Collection bbartifacts = new ArrayList<>();
for (Map.Entry entry : metadata.entrySet()) {
if (METADATA_TYPES_MAP.containsKey(entry.getKey())) {
@@ -573,32 +643,31 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
bbartifacts.add(bbart);
} catch (TskCoreException ex) {
// Log error and return to continue processing
- logger.log(Level.WARNING, String.format("Error creating or adding metadata artifact for file %s.", aFile.getParentPath() + aFile.getName()), ex); //NON-NLS
- return;
+ logger.log(Level.WARNING, String.format("Error creating or adding metadata artifact for file %s.", aFile.getParentPath() + aFile.getName()), ex); //NON-NLS
+ return;
}
if (!bbartifacts.isEmpty()) {
- try{
+ try {
Case.getCurrentCaseThrows().getSleuthkitCase().getBlackboard().postArtifacts(bbartifacts, moduleName);
} catch (NoCurrentCaseException | Blackboard.BlackboardException ex) {
// Log error and return to continue processing
- logger.log(Level.WARNING, String.format("Unable to post blackboard artifacts for file $s.", aFile.getParentPath() + aFile.getName()) , ex); //NON-NLS
+ logger.log(Level.WARNING, String.format("Unable to post blackboard artifacts for file $s.", aFile.getParentPath() + aFile.getName()), ex); //NON-NLS
return;
}
}
}
}
-
private BlackboardAttribute checkAttribute(String key, String value) {
String moduleName = KeywordSearchIngestModule.class.getName();
- if (!value.isEmpty() && value.charAt(0) != ' ') {
+ if (!value.isEmpty() && value.charAt(0) != ' ') {
if (METADATA_DATE_TYPES.contains(key)) {
- SimpleDateFormat metadataDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", US);
+ SimpleDateFormat metadataDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", US);
Long metadataDateTime = Long.valueOf(0);
try {
- String metadataDate = value.replaceAll("T"," ").replaceAll("Z", "");
+ String metadataDate = value.replaceAll("T", " ").replaceAll("Z", "");
Date usedDate = metadataDateFormat.parse(metadataDate);
- metadataDateTime = usedDate.getTime()/1000;
+ metadataDateTime = usedDate.getTime() / 1000;
return new BlackboardAttribute(METADATA_TYPES_MAP.get(key), moduleName, metadataDateTime);
} catch (ParseException ex) {
// catching error and displaying date that could not be parsed then will continue on.
@@ -609,12 +678,11 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
return new BlackboardAttribute(METADATA_TYPES_MAP.get(key), moduleName, value);
}
}
-
+
return null;
}
-
-
+
/**
* Pretty print the text extractor metadata.
*
@@ -639,7 +707,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
* Extract strings using heuristics from the file and add to index.
*
* @param aFile file to extract strings from, divide into chunks and
- * index
+ * index
*
* @return true if the file was text_ingested, false otherwise
*/
@@ -668,21 +736,24 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
/**
* Adds the file to the index. Detects file type, calls extractors, etc.
*
- * @param aFile File to analyze
+ * @param extractor The textExtractor to use with this file or empty
+ * if no extractor found.
+ * @param aFile File to analyze.
+ * @param mimeType The file mime type.
* @param indexContent False if only metadata should be text_ingested.
- * True if content and metadata should be index.
+ * True if content and metadata should be index.
*/
- private void indexFile(AbstractFile aFile, boolean indexContent) {
+ private void indexFile(Optional extractor, AbstractFile aFile, String mimeType, boolean indexContent) {
//logger.log(Level.INFO, "Processing AbstractFile: " + abstractFile.getName());
TskData.TSK_DB_FILES_TYPE_ENUM aType = aFile.getType();
/**
* Extract unicode strings from unallocated and unused blocks and
- * carved text files. The reason for performing string extraction
- * on these is because they all may contain multiple encodings which
- * can cause text to be missed by the more specialized text extractors
- * used below.
+ * carved text files. The reason for performing string extraction on
+ * these is because they all may contain multiple encodings which
+ * can cause text to be missed by the more specialized text
+ * extractors used below.
*/
if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
|| aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))
@@ -714,11 +785,10 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return;
}
- String fileType = fileTypeDetector.getMIMEType(aFile);
// we skip archive formats that are opened by the archive module.
// @@@ We could have a check here to see if the archive module was enabled though...
- if (ARCHIVE_MIME_TYPES.contains(fileType)) {
+ if (ARCHIVE_MIME_TYPES.contains(mimeType)) {
try {
if (context.fileIngestIsCancelled()) {
return;
@@ -741,11 +811,11 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (context.fileIngestIsCancelled()) {
return;
}
- if (fileType.equals(MimeTypes.OCTET_STREAM)) {
+ if (MimeTypes.OCTET_STREAM.equals(mimeType)) {
extractStringsAndIndex(aFile);
return;
}
- if (!extractTextAndIndex(aFile, extractedMetadata)) {
+ if (!extractTextAndIndex(extractor, aFile, extractedMetadata)) {
// Text extractor not found for file. Extract string only.
putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
} else {
@@ -773,7 +843,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
if (wasTextAdded == false) {
extractStringsAndIndex(aFile);
}
-
+
// Now that the indexing is complete, create the metadata artifact (if applicable).
// It is unclear why calling this from extractTextAndIndex() generates
// errors.
@@ -783,8 +853,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
/**
- * Adds the text file to the index given an encoding.
- * Returns true if indexing was successful and false otherwise.
+ * Adds the text file to the index given an encoding. Returns true if
+ * indexing was successful and false otherwise.
*
* @param aFile Text file to analyze
*/
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java
index 8ad969863b..826005f7d0 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettings.java
@@ -26,19 +26,35 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
/**
* Ingest job settings for the keywords search module.
*/
-final class KeywordSearchJobSettings implements IngestModuleIngestJobSettings {
+public final class KeywordSearchJobSettings implements IngestModuleIngestJobSettings {
private static final long serialVersionUID = 1L;
+
private HashSet namesOfEnabledKeywordLists;
private HashSet namesOfDisabledKeywordLists; // Added in version 1.1
+ /**
+ * These are nullable so that if no serialized setting, the setting can
+ * defer to legacy KeywordSearchSettings.
+ */
+ private Boolean ocrEnabled;
+ private Boolean limitedOCREnabled;
+
+ private boolean ocrOnly;
+
/**
* Constructs ingest job settings for the keywords search module.
*
* @param namesOfEnabledKeywordLists A list of enabled keywords lists.
*/
KeywordSearchJobSettings(List namesOfEnabledKeywordLists) {
- this(namesOfEnabledKeywordLists, new ArrayList());
+ this.namesOfEnabledKeywordLists = new HashSet<>(namesOfEnabledKeywordLists);
+ this.namesOfDisabledKeywordLists = new HashSet<>();
+
+ // explicitly set to default value
+ this.ocrEnabled = null;
+ this.limitedOCREnabled = null;
+ this.ocrOnly = false;
}
/**
@@ -46,10 +62,88 @@ final class KeywordSearchJobSettings implements IngestModuleIngestJobSettings {
*
* @param namesOfEnabledKeywordLists A list of enabled keywords lists.
* @param namesOfDisabledKeywordLists A list of disabled keywords lists.
+ * @param ocrEnabled Whether or not OCR is enabled for
+ * keyword search.
+ * @param limitedOCREnabled If true, OCR is to be performed only
+ * on images larger than 100KB.
+ * @param ocrOnly True if keyword search ingest should
+ * be solely limited to OCR.
*/
- KeywordSearchJobSettings(List namesOfEnabledKeywordLists, List namesOfDisabledKeywordLists) {
+ KeywordSearchJobSettings(List namesOfEnabledKeywordLists, List namesOfDisabledKeywordLists, boolean ocrEnabled, boolean limitedOCREnabled, boolean ocrOnly) {
this.namesOfEnabledKeywordLists = new HashSet<>(namesOfEnabledKeywordLists);
this.namesOfDisabledKeywordLists = new HashSet<>(namesOfDisabledKeywordLists);
+ this.ocrEnabled = ocrEnabled;
+ this.limitedOCREnabled = limitedOCREnabled;
+ this.ocrOnly = ocrOnly;
+ }
+
+ /**
+ * Whether or not OCR is enabled for keyword search.
+ *
+ * @return Whether or not OCR is enabled for keyword search.
+ */
+ @SuppressWarnings("deprecation")
+ public boolean isOCREnabled() {
+ if (ocrEnabled == null) {
+ ocrEnabled = KeywordSearchSettings.getOcrOption();
+ }
+
+ return ocrEnabled;
+ }
+
+ /**
+ * Sets whether or not OCR is enabled for keyword search.
+ *
+ * @param ocrEnabled Whether or not OCR is enabled for keyword search.
+ */
+ public void setOCREnabled(boolean ocrEnabled) {
+ this.ocrEnabled = ocrEnabled;
+ }
+
+ /**
+ * Returns true if OCR is to be performed only on images larger than 100KB.
+ * May defer to KeywordSearchSettings if no setting serialized.
+ *
+ * @return If true, OCR is to be performed only on images larger than 100KB.
+ */
+ @SuppressWarnings("deprecation")
+ boolean isLimitedOCREnabled() {
+ if (limitedOCREnabled == null) {
+ limitedOCREnabled = KeywordSearchSettings.getLimitedOcrOption();
+ }
+
+ return limitedOCREnabled;
+ }
+
+ /**
+ * Sets whether or not OCR should be performed only on images larger than
+ * 100KB.
+ *
+ * @param limitedOCREnabled Whether or not OCR should be performed only on
+ * images larger than 100KB.
+ */
+ void setLimitedOCREnabled(boolean limitedOCREnabled) {
+ this.limitedOCREnabled = limitedOCREnabled;
+ }
+
+ /**
+ * Returns true if keyword search ingest should be solely limited to OCR.
+ *
+ * @return True if keyword search ingest should be solely limited to OCR.
+ */
+ boolean isOCROnly() {
+ return ocrOnly;
+ }
+
+ /**
+ * Sets whether or not keyword search ingest should be solely limited to
+ * OCR.
+ *
+ * @param ocrOnly Whether or not keyword search ingest should be solely
+ * limited to OCR.
+ */
+ void setOCROnly(boolean ocrOnly) {
+ this.ocrOnly = ocrOnly;
}
/**
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.form
index ac3cdd5782..0294690848 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.form
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.form
@@ -20,30 +20,34 @@
-
-
+
+
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -54,18 +58,24 @@
+
+
-
-
-
-
+
+
-
+
-
+
+
+
+
+
+
+
@@ -96,8 +106,15 @@
-
-
+
+
+
+
+
+
+
+
+
@@ -150,5 +167,36 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.java
index a71485cbb1..6bf036000a 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchJobSettingsPanel.java
@@ -18,19 +18,17 @@
*/
package org.sleuthkit.autopsy.keywordsearch;
-import java.awt.Component;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import javax.swing.JLabel;
import javax.swing.JTable;
import javax.swing.ListSelectionModel;
import javax.swing.table.AbstractTableModel;
-import javax.swing.table.DefaultTableCellRenderer;
import javax.swing.table.TableColumn;
+import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
import org.sleuthkit.autopsy.guiutils.SimpleTableCellRenderer;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
@@ -41,17 +39,17 @@ import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.StringsExtr
* Ingest job settings panel for keyword search file ingest modules.
*/
@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives
-public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSettingsPanel implements PropertyChangeListener {
-
+public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSettingsPanel implements PropertyChangeListener {
private final KeywordListsTableModel tableModel = new KeywordListsTableModel();
private final List keywordListNames = new ArrayList<>();
private final Map keywordListStates = new HashMap<>();
private final XmlKeywordSearchList keywordListsManager = XmlKeywordSearchList.getCurrent();
+
KeywordSearchJobSettingsPanel(KeywordSearchJobSettings initialSettings) {
- initializeKeywordListSettings(initialSettings);
initComponents();
customizeComponents();
+ initializeKeywordListSettings(initialSettings);
}
private void initializeKeywordListSettings(KeywordSearchJobSettings settings) {
@@ -63,6 +61,23 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
keywordListNames.add(listName);
keywordListStates.put(listName, settings.keywordListIsEnabled(listName));
}
+
+ ocrCheckBox.setSelected(settings.isOCREnabled());
+ limitedOcrCheckbox.setSelected(settings.isLimitedOCREnabled());
+ ocrOnlyCheckbox.setSelected(settings.isOCROnly());
+
+ handleOcrEnabled(settings.isOCREnabled());
+ }
+
+ /**
+ * Handles setting enabled state of checkbox.
+ * @param ocrEnabled Whether or not the ocr setting is enabled.
+ */
+ private void handleOcrEnabled(boolean ocrEnabled) {
+ boolean platformSupported = PlatformUtil.isWindowsOS() && PlatformUtil.is64BitOS();
+ ocrCheckBox.setEnabled(platformSupported);
+ limitedOcrCheckbox.setEnabled(platformSupported && ocrEnabled);
+ ocrOnlyCheckbox.setEnabled(platformSupported && ocrEnabled);
}
private void customizeComponents() {
@@ -71,6 +86,10 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
displayEncodings();
keywordListsManager.addPropertyChangeListener(this);
languagesLabel.setText("" + org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.languagesLabel.text") + ""); // NOI18N NON-NLS
+
+ // the gui builder does not explicitly set these to false.
+ listsTable.setShowHorizontalLines(false);
+ listsTable.setShowVerticalLines(false);
}
private void customizeKeywordListsTable() {
@@ -174,7 +193,8 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
disabledListNames.add(listName);
}
}
- return new KeywordSearchJobSettings(enabledListNames, disabledListNames);
+ return new KeywordSearchJobSettings(enabledListNames, disabledListNames,
+ this.ocrCheckBox.isSelected(), this.limitedOcrCheckbox.isSelected(), this.ocrOnlyCheckbox.isSelected());
}
void reset(KeywordSearchJobSettings newSettings) {
@@ -241,6 +261,9 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
languagesValLabel = new javax.swing.JLabel();
encodingsLabel = new javax.swing.JLabel();
keywordSearchEncodings = new javax.swing.JLabel();
+ ocrCheckBox = new javax.swing.JCheckBox();
+ limitedOcrCheckbox = new javax.swing.JCheckBox();
+ ocrOnlyCheckbox = new javax.swing.JCheckBox();
setPreferredSize(new java.awt.Dimension(300, 170));
@@ -256,8 +279,9 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
}
));
- listsTable.setShowHorizontalLines(false);
- listsTable.setShowVerticalLines(false);
+ listsTable.setMaximumSize(new java.awt.Dimension(32767, 32767));
+ listsTable.setMinimumSize(new java.awt.Dimension(20, 200));
+ listsTable.setPreferredSize(null);
listsScrollPane.setViewportView(listsTable);
listsTable.setDefaultRenderer(String.class, new SimpleTableCellRenderer());
@@ -275,29 +299,54 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
keywordSearchEncodings.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.keywordSearchEncodings.text")); // NOI18N
+ ocrCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.ocrCheckBox.text")); // NOI18N
+ ocrCheckBox.addActionListener(new java.awt.event.ActionListener() {
+ public void actionPerformed(java.awt.event.ActionEvent evt) {
+ ocrCheckBoxActionPerformed(evt);
+ }
+ });
+
+ limitedOcrCheckbox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.limitedOcrCheckbox.text")); // NOI18N
+ limitedOcrCheckbox.setVerticalTextPosition(javax.swing.SwingConstants.TOP);
+ limitedOcrCheckbox.addActionListener(new java.awt.event.ActionListener() {
+ public void actionPerformed(java.awt.event.ActionEvent evt) {
+ limitedOcrCheckboxActionPerformed(evt);
+ }
+ });
+
+ ocrOnlyCheckbox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchJobSettingsPanel.class, "KeywordSearchJobSettingsPanel.ocrOnlyCheckbox.text")); // NOI18N
+ ocrOnlyCheckbox.addActionListener(new java.awt.event.ActionListener() {
+ public void actionPerformed(java.awt.event.ActionEvent evt) {
+ ocrOnlyCheckboxActionPerformed(evt);
+ }
+ });
+
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
- layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+ layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
- .addGroup(layout.createSequentialGroup()
- .addComponent(listsScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE)
- .addContainerGap())
- .addGroup(layout.createSequentialGroup()
- .addGap(10, 10, 10)
- .addComponent(languagesValLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 274, javax.swing.GroupLayout.PREFERRED_SIZE)
- .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addComponent(languagesLabel, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addGroup(layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+ .addComponent(listsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 316, Short.MAX_VALUE)
.addComponent(titleLabel)
.addGroup(layout.createSequentialGroup()
.addComponent(encodingsLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
- .addComponent(keywordSearchEncodings)))
- .addGap(0, 0, Short.MAX_VALUE))))
+ .addComponent(keywordSearchEncodings))
+ .addGroup(layout.createSequentialGroup()
+ .addGap(10, 10, 10)
+ .addComponent(languagesValLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 274, javax.swing.GroupLayout.PREFERRED_SIZE))
+ .addComponent(ocrCheckBox)
+ .addGroup(layout.createSequentialGroup()
+ .addGap(21, 21, 21)
+ .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+ .addComponent(ocrOnlyCheckbox)
+ .addComponent(limitedOcrCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 288, javax.swing.GroupLayout.PREFERRED_SIZE))))
+ .addContainerGap())))
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
@@ -305,25 +354,48 @@ public final class KeywordSearchJobSettingsPanel extends IngestModuleIngestJobSe
.addGap(7, 7, 7)
.addComponent(titleLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
- .addComponent(listsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 41, Short.MAX_VALUE)
- .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
- .addComponent(languagesLabel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
+ .addComponent(listsScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
+ .addComponent(languagesLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 13, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(languagesValLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(encodingsLabel)
.addComponent(keywordSearchEncodings))
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
+ .addComponent(ocrCheckBox)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
+ .addComponent(ocrOnlyCheckbox)
+ .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
+ .addComponent(limitedOcrCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addContainerGap())
);
}// //GEN-END:initComponents
+
+ private void ocrCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ocrCheckBoxActionPerformed
+ handleOcrEnabled(ocrCheckBox.isSelected());
+ firePropertyChange(KeywordSearchOptionsPanelController.PROP_CHANGED, null, null);
+ }//GEN-LAST:event_ocrCheckBoxActionPerformed
+
+ private void limitedOcrCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_limitedOcrCheckboxActionPerformed
+ firePropertyChange(KeywordSearchOptionsPanelController.PROP_CHANGED, null, null);
+ }//GEN-LAST:event_limitedOcrCheckboxActionPerformed
+
+ private void ocrOnlyCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_ocrOnlyCheckboxActionPerformed
+ // TODO add your handling code here:
+ }//GEN-LAST:event_ocrOnlyCheckboxActionPerformed
+
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JLabel encodingsLabel;
private javax.swing.JLabel keywordSearchEncodings;
private javax.swing.JLabel languagesLabel;
private javax.swing.JLabel languagesValLabel;
+ private javax.swing.JCheckBox limitedOcrCheckbox;
private javax.swing.JScrollPane listsScrollPane;
private javax.swing.JTable listsTable;
+ private javax.swing.JCheckBox ocrCheckBox;
+ private javax.swing.JCheckBox ocrOnlyCheckbox;
private javax.swing.JLabel titleLabel;
// End of variables declaration//GEN-END:variables
}
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java
index 0db0b30684..b7057a2c89 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java
@@ -133,20 +133,14 @@ class KeywordSearchSettings {
ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, key, val);
}
- /**
- * Save OCR setting to permanent storage
- *
- * @param enabled Is OCR enabled?
- */
- static void setOcrOption(boolean enabled) {
- ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, OCR_ENABLED, (enabled ? "true" : "false")); //NON-NLS
- }
-
/**
* Get OCR setting from permanent storage
*
* @return Is OCR enabled?
+ *
+ * @deprecated Please use KeywordSearchJobSettings instead.
*/
+ @Deprecated
static boolean getOcrOption() {
if (ModuleSettings.settingExists(PROPERTIES_OPTIONS, OCR_ENABLED)) {
return ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, OCR_ENABLED).equals("true"); //NON-NLS
@@ -155,6 +149,24 @@ class KeywordSearchSettings {
}
}
+ /**
+ * Gets the limited OCR flag to indicate if OCR should be limited to larger
+ * images and images which were extracted from documents.
+ *
+ * @return Flag indicating if limited OCR is enabled. True if OCR should be
+ * limited, false otherwise.
+ *
+ * @deprecated Please use KeywordSearchJobSettings instead.
+ */
+ @Deprecated
+ static boolean getLimitedOcrOption() {
+ if (ModuleSettings.settingExists(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) {
+ return ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED).equals("true"); //NON-NLS
+ } else {
+ return LIMITED_OCR_ENABLED_DEFAULT;
+ }
+ }
+
static void setShowSnippets(boolean showSnippets) {
ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, SHOW_SNIPPETS, (showSnippets ? "true" : "false")); //NON-NLS
}
@@ -246,45 +258,10 @@ class KeywordSearchSettings {
logger.log(Level.INFO, "No configuration for UTF16 found, generating defaults..."); //NON-NLS
KeywordSearchSettings.setStringExtractOption(StringsExtractOptions.EXTRACT_UTF16.toString(), Boolean.TRUE.toString());
}
- //setting OCR default (disabled by default)
- if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, OCR_ENABLED)) {
- logger.log(Level.INFO, "No configuration for OCR found, generating defaults..."); //NON-NLS
- KeywordSearchSettings.setOcrOption(OCR_ENABLED_DEFAULT);
- }
- //setting OCR default (disabled by default)
- if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) {
- logger.log(Level.INFO, "No configuration for OCR found, generating defaults..."); //NON-NLS
- KeywordSearchSettings.setLimitedOcrOption(LIMITED_OCR_ENABLED_DEFAULT);
- }
//setting default Latin-1 Script
if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_SCRIPTS, SCRIPT.LATIN_1.name())) {
logger.log(Level.INFO, "No configuration for Scripts found, generating defaults..."); //NON-NLS
ModuleSettings.setConfigSetting(KeywordSearchSettings.PROPERTIES_SCRIPTS, SCRIPT.LATIN_1.name(), Boolean.toString(true));
}
}
-
- /**
- * Enables the limiting OCR to be run on larger images and images which were
- * extracted from documents.
- *
- * @param enabled Flag indicating if OCR is enabled.
- */
- static void setLimitedOcrOption(boolean enabled) {
- ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED, (enabled ? "true" : "false")); //NON-NLS
- }
-
- /**
- * Gets the limited OCR flag to indicate if OCR should be limited to larger
- * images and images which were extracted from documents.
- *
- * @return Flag indicating if limited OCR is enabled. True if OCR should be
- * limited, false otherwise..
- */
- static boolean getLimitedOcrOption() {
- if (ModuleSettings.settingExists(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED)) {
- return ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, LIMITED_OCR_ENABLED).equals("true"); //NON-NLS
- } else {
- return LIMITED_OCR_ENABLED_DEFAULT;
- }
- }
}
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
index ee20647466..e5c3f8ae03 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
@@ -466,18 +466,4 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
throw new TskCoreException(ex.getCause().getMessage(), ex);
}
}
-
- /**
- * A flag to enable or disable OCR on all future text indexing. Also sets the
- * the "Limited OCR" functionality accordingly.
- *
- * @param state Boolean flag to enable/disable OCR. Set to True to enable
- * OCR, or False to disable it.
- */
- @Override
- public void changeOcrState(boolean state) {
- KeywordSearchSettings.setOcrOption(state);
- KeywordSearchSettings.setLimitedOcrOption(state);
- }
-
}
diff --git a/NEWS.txt b/NEWS.txt
index d9e2f99b90..3ab13cfab3 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -5,12 +5,18 @@ Data Source Management:
- The main tree viewer can be configured to group by person and host.
OS Accounts:
-- Operating System (OS) accounts and realms are their own data types and not generic artifacts.
+- Operating System (OS) accounts and realms are their own data types and no longer generic artifacts.
- OS Accounts are created for Windows accounts found in the registry. Domain-scoped realms are not fully detected yet.
- NTFS files are associated with OS Accounts by SID.
- The Recent Activity module associates artifacts with OS Accounts based on SID or path of database. Other modules still need to be updated.
- OS accounts appear in a dedicated sub-tree of the main tree view and their properties can be viewed in the results view.
-- A new content viewer in the lower right area of the main window was built to display OS account data for the item selected in the result view.
+- A new content viewer in the lower right area of the main window was built to display OS account data for the item selected in the results view.
+
+Analysis Result and Data Artifacts
+- All modules make either Analysis Results or Data Artifacts instead of “Blackboard Artifacts.”
+- New “Analysis Result” content viewer shows the results for a given file and its score.
+- The tabular results viewer shows an icon for the aggregate score of a file.
+- The tree organizes results into "Analysis Results" and "Data Artifacts" instead of simply “Results.”
Discovery UI:
- Domain categorization and account types are displayed in Domain Discovery results.
@@ -22,17 +28,40 @@ Ingest Modules:
- Parsing of iLEAPP and aLEAPP output was expanded to create communication relationships which can be displayed in the Communications UI.
- EML email parsing handles EML messages that are attachments (and have their own attachments).
- Domain categorization within Recent Activity can be customized by user-defined rules that can be imported and exported.
+- Account IDs and Installed Applications are added to the Central Repository.
+- Keyword search can be configured to only do OCR and skip non-OCR files.
Miscellaneous:
- A “Reset Windows” feature was created to help redock windows.
- A case-insensitive wordlist of all words in the keyword search index can be exported as a text document.
- Information from the Data Source Summary panels can be exported as an Excel spreadsheet.
- More artifacts are added to the timeline and artifacts with multiple time-based attributes are mapped to multiple timeline events.
-- The Auto Ingest Dashboard is resizable.
- Added option to only perform optical character recognition on certain file types.
- Heap dumps can be saved to a custom location.
+- More detailed error messages about encrypted disks when they are added.
+- Added file size filter to Ingest Filters.
+
+Performance:
+- Keyword search does not make an explicit commit for each report if ingest is running.
+- Language ID is performed on a small subset of a file instead of the entire file.
+- Recent Activity is more efficient because of TSK changes to file searching (using extension).
+- Embedded file extractor module has been made faster by doing file typing in memory and adding extracted files in batches.
+- Moved Content Viewers setNode() and isSupported()/isPreferred() code to background threads.
+- Moved Data Source Summary Panel population code to background threads.
+- Moved Node/Tree queries to background threads.
+
+Bug Fixes:
+- Fixed embedded file extractor file name escaping bug.
+- Detect VHD files by signature and not extension.
+- Fixed iLEAPP path error.
+- Content viewers UIs are more consistent.
- Assorted bug fixes are included.
+Auto Ingest:
+- The Auto Ingest Dashboard is resizable.
+- Get thread dumps from AID
+- Added beta pause feature that pauses auto ingest for a set amount of time at a scheduled date and time.
+
---------------- VERSION 4.18.0 --------------
Keyword Search:
- A major upgrade from Solr 4 to Solr 8.6.3. Single user cases continue to use the embedded server.
diff --git a/RecentActivity/manifest.mf b/RecentActivity/manifest.mf
index 76b6b1ce6f..03c6ce986b 100644
--- a/RecentActivity/manifest.mf
+++ b/RecentActivity/manifest.mf
@@ -1,6 +1,6 @@
Manifest-Version: 1.0
OpenIDE-Module: org.sleuthkit.autopsy.recentactivity/6
-OpenIDE-Module-Implementation-Version: 18
+OpenIDE-Module-Implementation-Version: 19
OpenIDE-Module-Layer: org/sleuthkit/autopsy/recentactivity/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/recentactivity/Bundle.properties
OpenIDE-Module-Requires:
diff --git a/RecentActivity/nbproject/project.xml b/RecentActivity/nbproject/project.xml
index 8fc5e13b53..11dc55cab6 100644
--- a/RecentActivity/nbproject/project.xml
+++ b/RecentActivity/nbproject/project.xml
@@ -60,7 +60,7 @@
10
- 10.23
+ 10.24
diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java
index 092cea047d..e3df208180 100644
--- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java
+++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java
@@ -1995,7 +1995,7 @@ class ExtractRegistry extends Extract {
} else {
osAccount = optional.get();
if (userName != null && !userName.isEmpty()) {
- OsAccountUpdateResult updateResult= accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, userName, null, host);
+ OsAccountUpdateResult updateResult= accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, userName, domainName.isEmpty() ? null : domainName, host);
osAccount = updateResult.getUpdatedAccount().orElse(osAccount);
}
}
@@ -2187,7 +2187,7 @@ class ExtractRegistry extends Extract {
accountMgr.addExtendedOsAccountAttributes(osAccount, attributes);
// update the loginname
- accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, loginName, null, host);
+ accountMgr.updateCoreWindowsOsAccountAttributes(osAccount, null, loginName, domainName.isEmpty() ? null : domainName, host);
// update other standard attributes - fullname, creationdate
accountMgr.updateStandardOsAccountAttributes(osAccount, fullName, null, null, creationTime);
diff --git a/TSKVersion.xml b/TSKVersion.xml
index c0fece9373..0c5ef49ada 100644
--- a/TSKVersion.xml
+++ b/TSKVersion.xml
@@ -1,3 +1,3 @@
-
+
diff --git a/Testing/nbproject/project.xml b/Testing/nbproject/project.xml
index acf97c4ddf..141594e95a 100644
--- a/Testing/nbproject/project.xml
+++ b/Testing/nbproject/project.xml
@@ -47,7 +47,7 @@
10
- 10.23
+ 10.24
diff --git a/docs/doxygen-user/Doxyfile b/docs/doxygen-user/Doxyfile
index d00e28635a..6074fa5e59 100644
--- a/docs/doxygen-user/Doxyfile
+++ b/docs/doxygen-user/Doxyfile
@@ -38,7 +38,7 @@ PROJECT_NAME = "Autopsy User Documentation"
# could be handy for archiving the generated documentation or if some version
# control system is used.
-PROJECT_NUMBER = 4.18.0
+PROJECT_NUMBER = 4.19.0
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
@@ -1025,7 +1025,7 @@ GENERATE_HTML = YES
# The default directory is: html.
# This tag requires that the tag GENERATE_HTML is set to YES.
-HTML_OUTPUT = 4.18.0
+HTML_OUTPUT = 4.19.0
# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
# generated HTML page (for example: .htm, .php, .asp).
diff --git a/docs/doxygen-user/auto_ingest_administration.dox b/docs/doxygen-user/auto_ingest_administration.dox
index 089a34d474..029a3d47f8 100644
--- a/docs/doxygen-user/auto_ingest_administration.dox
+++ b/docs/doxygen-user/auto_ingest_administration.dox
@@ -34,7 +34,15 @@ With the admin file in place, the user can right-click on jobs in each of the ta
\image html AutoIngest/admin_jobs_panel.png
-In the Running Jobs tables, the ingest progress can be viewed and the current job can be cancelled. Note that cancellation can take some time.
+You can also selectively enable \ref keyword_search_ocr_config "optical character recognition (OCR)" on a case-by-case basis. This will override the normal Keyword Search settings for every job in the case. Jobs in progress will not be affected. To enable OCR for a case, right-click on a job from that case in the Pending Jobs table and select "Enable OCR For This Case".
+
+\image html admin_jobs_ocr1.png
+
+Once enabled, a green checkmark will appear in the OCR column next to every pending job for that case.
+
+\image html admin_jobs_ocr2.png
+
+In the Running Jobs tables, the ingest progress can be viewed and the current job can be cancelled. Note that cancellation can take some time. You can also generate a thread dump if you suspect ingest may be stuck.
\image html AutoIngest/admin_jobs_cancel.png
@@ -42,9 +50,11 @@ In the Completed Jobs table, the user can reprocess a job (generally useful when
\image html AutoIngest/admin_jobs_completed.png
+
+
\section auto_ingest_admin_nodes_panel Auto Ingest Nodes Panel
-The Nodes panel displays the status of every online auto ingest node. Additionally, an admin can pause or resume a node, or shut down a node entirely (i.e., exit the Autopsy app).
+The Nodes panel displays the status of every online auto ingest node. Additionally, an admin can pause or resume a node, generate a thread dump, or shut down a node entirely (i.e., exit the Autopsy app).
\image html AutoIngest/admin_nodes_panel.png
diff --git a/docs/doxygen-user/images/AutoIngest/admin_jobs_cancel.png b/docs/doxygen-user/images/AutoIngest/admin_jobs_cancel.png
index ba3fb81691..d8cbd4deee 100644
Binary files a/docs/doxygen-user/images/AutoIngest/admin_jobs_cancel.png and b/docs/doxygen-user/images/AutoIngest/admin_jobs_cancel.png differ
diff --git a/docs/doxygen-user/images/AutoIngest/admin_jobs_ocr1.png b/docs/doxygen-user/images/AutoIngest/admin_jobs_ocr1.png
new file mode 100644
index 0000000000..a0db003ca6
Binary files /dev/null and b/docs/doxygen-user/images/AutoIngest/admin_jobs_ocr1.png differ
diff --git a/docs/doxygen-user/images/AutoIngest/admin_jobs_ocr2.png b/docs/doxygen-user/images/AutoIngest/admin_jobs_ocr2.png
new file mode 100644
index 0000000000..85aabf1154
Binary files /dev/null and b/docs/doxygen-user/images/AutoIngest/admin_jobs_ocr2.png differ
diff --git a/docs/doxygen-user/images/AutoIngest/admin_jobs_panel.png b/docs/doxygen-user/images/AutoIngest/admin_jobs_panel.png
index 32dfa89cdd..25ad7b1982 100644
Binary files a/docs/doxygen-user/images/AutoIngest/admin_jobs_panel.png and b/docs/doxygen-user/images/AutoIngest/admin_jobs_panel.png differ
diff --git a/docs/doxygen-user/images/AutoIngest/admin_nodes_panel.png b/docs/doxygen-user/images/AutoIngest/admin_nodes_panel.png
index 08488323dd..220b5d65a4 100644
Binary files a/docs/doxygen-user/images/AutoIngest/admin_nodes_panel.png and b/docs/doxygen-user/images/AutoIngest/admin_nodes_panel.png differ
diff --git a/docs/doxygen-user/images/solr/solr_comment_out_updateLog.jpg b/docs/doxygen-user/images/solr/solr_comment_out_updateLog.jpg
new file mode 100644
index 0000000000..90737515fa
Binary files /dev/null and b/docs/doxygen-user/images/solr/solr_comment_out_updateLog.jpg differ
diff --git a/docs/doxygen-user/images/solr/solr_config_autocommit.jpg b/docs/doxygen-user/images/solr/solr_config_autocommit.jpg
new file mode 100644
index 0000000000..5e6cc7a9bf
Binary files /dev/null and b/docs/doxygen-user/images/solr/solr_config_autocommit.jpg differ
diff --git a/docs/doxygen-user/images/solr/solr_transaction_log_errors.jpg b/docs/doxygen-user/images/solr/solr_transaction_log_errors.jpg
new file mode 100644
index 0000000000..4328068b57
Binary files /dev/null and b/docs/doxygen-user/images/solr/solr_transaction_log_errors.jpg differ
diff --git a/docs/doxygen-user/multi-user/installSolr.dox b/docs/doxygen-user/multi-user/installSolr.dox
index d8a4690d58..dd380e6eef 100644
--- a/docs/doxygen-user/multi-user/installSolr.dox
+++ b/docs/doxygen-user/multi-user/installSolr.dox
@@ -207,7 +207,7 @@ Solr creates two types of data that need to be backed up:
- In a default installation that data is stored in \c "C:\solr-8.6.3\server\solr zoo_data" (assuming that the Solr package ZIP was extracted into \c "C:\solr-8.6.3" directory).
-\section Troubleshooting
+\section troubleshooting Troubleshooting / Performance Tuning
\subsection install_solr_delayed_start Delayed Start Problems With Large Number Of Solr Collections
@@ -287,4 +287,36 @@ Some notes:
+\subsection solr_commit_tuning Tuning Solr Server Commit Operations
+
+When Autopsy is running in a multi-user cluster environment with multiple auto ingest nodes, it may be beneficial to tune the configuration of Solr "commit" operations.
+
+Solr has two types of "commits" – hard commits and soft commits. These are explained in detail the following link: https://lucidworks.com/post/understanding-transaction-logs-softcommit-and-commit-in-sorlcloud/
+
+In short:
+
+- Hard commits flush newly indexed documents from RAM to Solr index on disk.
+
- Depending on configuration, hard commits may or may not make the newly indexed documents "visible" to search.
+
- Soft commits do not flush newly indexed documents to disk but they make the newly indexed documents "visible" to search.
+
+
+By default (when using AutopsyConfig) the Solr servers are performing a "hard" commit every 5 minutes and are also making the newly indexed documents "visible" to search. These operations can be costly when performed on a large index which is located on a shared network drive. In this stuation it can be very beneficial to modify Solr configuration (located in \c "SOLR_INSTALLATION_DIECTORY\server\solr\configsets\AutopsyConfig\conf\solrconfig.xml") with the following changes:
+
+- Modify "hard commits" to still flush the newly created documents every 5 minutes but not make them "visible". This is accomplished by setting “openSearcher" to "false" in the "autoCommit" section of Solr configuration file.
+
- Enable "soft commits" to be performed every 30 minutes thus making the newly indexed documents "visible" to search every 30 minutes. This is accomplished by enabling the "autoSoftCommit" section of Solr configuration file. The downside is that it may take up to 30 minutes to be able to search the latest document. Keep in mind that this only affects the scenario where an examiner is searching a case while the ingest is still ongoing. Autopsy automatically performs a commit after the ingest is complete so all the documents are immediately visible at that time.
+
+
+The following image shows the Solr configuration changes discussed above:
+
+\image html solr_config_autocommit.jpg
+
+Until a hard commit is performed, by default Solr also maintains a transaction log which contains the raw text of every newly indexed document since last hard commit. When the Solr index is located on a network share, this again can be a very costly operation. Transaction logs can be disabled by commenting out the "updateLog" section of Solr configuration file:
+
+\image html solr_comment_out_updateLog.jpg
+
+Keep in mind that this has an unfortunate side effect of creating CommitTracker errors in Solr logs and Solr admin console. These errors can be ignored if transaction log has been intentionally disabled.
+
+\image html solr_transaction_log_errors.jpg
+
+
*/
diff --git a/docs/doxygen/Doxyfile b/docs/doxygen/Doxyfile
index fba973919f..7c230015ff 100644
--- a/docs/doxygen/Doxyfile
+++ b/docs/doxygen/Doxyfile
@@ -38,7 +38,7 @@ PROJECT_NAME = "Autopsy"
# could be handy for archiving the generated documentation or if some version
# control system is used.
-PROJECT_NUMBER = 4.18.0
+PROJECT_NUMBER = 4.19.0
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears a the top of each page and should give viewer a
@@ -1066,7 +1066,7 @@ GENERATE_HTML = YES
# The default directory is: html.
# This tag requires that the tag GENERATE_HTML is set to YES.
-HTML_OUTPUT = api-docs/4.18.0/
+HTML_OUTPUT = api-docs/4.19.0/
# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
# generated HTML page (for example: .htm, .php, .asp).
diff --git a/nbproject/project.properties b/nbproject/project.properties
index 2b71bc4a1f..16be0556f0 100644
--- a/nbproject/project.properties
+++ b/nbproject/project.properties
@@ -4,7 +4,7 @@ app.title=Autopsy
### lowercase version of above
app.name=${branding.token}
### if left unset, version will default to today's date
-app.version=4.18.0
+app.version=4.19.0
### build.type must be one of: DEVELOPMENT, RELEASE
#build.type=RELEASE
build.type=DEVELOPMENT
diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml
index 318b4e36ee..333bbf6578 100644
--- a/thunderbirdparser/nbproject/project.xml
+++ b/thunderbirdparser/nbproject/project.xml
@@ -54,7 +54,7 @@
10
- 10.23
+ 10.24
diff --git a/unix_setup.sh b/unix_setup.sh
index 40b1d858b4..6554689384 100644
--- a/unix_setup.sh
+++ b/unix_setup.sh
@@ -5,7 +5,7 @@
# NOTE: update_sleuthkit_version.pl updates this value and relies
# on it keeping the same name and whitespace. Don't change it.
-TSK_VERSION=4.10.2
+TSK_VERSION=4.11.0
# In the beginning...